Browse Source

Finished price and date extrapolation

master
Patrick Gaskin 2 years ago
commit
d217a7ba1f
Signed by: geek1011 GPG Key ID: A2FD79F68A2AB707
  1. 3
      go.mod
  2. BIN
      graph.png
  3. 110
      main.go
  4. 67
      util.go

3
go.mod

@ -0,0 +1,3 @@
module dataextract2
go 1.12

BIN
graph.png

Binary file not shown.

After

Width:  |  Height:  |  Size: 552 KiB

110
main.go

@ -0,0 +1,110 @@
package main
import (
"fmt"
"image"
"math"
"time"
)
func main() {
// color checks for graphs generated by camelcamelcamel. Currently only supports
// Amazon prices (green line).
var (
isLineColor = colorMatcher(99, 168, 94)
isMaxLineColor = colorMatcher(194, 68, 68)
isMinLineColor = colorMatcher(119, 195, 107)
isGridColor = colorMatcher(215, 215, 214)
isBgColor = colorMatcher(255, 255, 254)
)
maxPrice, minPrice := 59.99, 9.99
startDate, endDate := time.Date(2015, 10, 29, 0, 0, 0, 0, time.Local), time.Date(2019, 05, 15, 0, 0, 0, 0, time.Local)
_, _, _, _, _, _, _, _, _ = isLineColor, isMaxLineColor, isMinLineColor, isGridColor, isBgColor, maxPrice, minPrice, startDate, endDate
img := must(loadPNG("graph.png")).(image.Image)
graphBounds := *getGraphBounds(img, isGridColor)
priceExtrapolator := getPriceExtrapolator(img, graphBounds, maxPrice, minPrice, isMaxLineColor, isMinLineColor)
dateExtrapolator := getDateExtrapolator(img, graphBounds, startDate, endDate)
fmt.Println(graphBounds) // (54,16)-(15945,7951)
fmt.Println(priceExtrapolator(3127)) // 39.99
fmt.Println(priceExtrapolator(16)) // 59.99
fmt.Println(priceExtrapolator(4682)) // 30.00
fmt.Println(priceExtrapolator(6862)) // 15.99
fmt.Println(priceExtrapolator(7795)) // 9.99
fmt.Println(dateExtrapolator(55)) // Oct 29, 2015
fmt.Println(dateExtrapolator(1636)) // Mar 6, 2016
fmt.Println(dateExtrapolator(3230)) // Jul 14, 2016
fmt.Println(dateExtrapolator(4824)) // Nov 20, 2016
fmt.Println(dateExtrapolator(6418)) // Mar 30, 2017
fmt.Println(dateExtrapolator(14387)) // Jan 8, 2019
}
// getGraphBounds returns the detected graph bounds (based on lines of the grid
// color at least 75% of the image width).
func getGraphBounds(img image.Image, fn colorMatcherFunc) *image.Rectangle {
var yTop, yBottom, xLeft, xRight int
for y := 0; y < img.Bounds().Max.Y; y++ {
if pctColor(img, fn, y, false) > 0.75 {
yTop = y
break
}
}
for y := img.Bounds().Max.Y; y > 0; y-- {
if pctColor(img, fn, y, false) > 0.75 {
yBottom = y
break
}
}
for x := 0; x < img.Bounds().Max.X; x++ {
if pctColor(img, fn, x, true) > 0.75 {
xLeft = x
break
}
}
for x := img.Bounds().Max.X; x > 0; x-- {
if pctColor(img, fn, x, true) > 0.75 {
xRight = x
break
}
}
if yTop == 0 || yBottom == 0 || xLeft == 0 || xRight == 0 {
return nil
}
r := image.Rect(xLeft, yTop, xRight, yBottom)
return &r
}
// getPriceExtrapolator gets an extrapolator to extract the price from the image
// based on a y value relative to the image (NOT the grid).
func getPriceExtrapolator(img image.Image, graphBounds image.Rectangle, maxPrice, minPrice float64, maxColor, minColor colorMatcherFunc) func(y int) float64 {
// dashed line, so threshold is 0.4; the line is on bottom of plotted line, so subtract 1.
var maxY, minY int
for y := graphBounds.Min.Y; y < graphBounds.Max.Y; y++ {
if maxY == 0 && pctColor(img, maxColor, y, false) > 0.4 {
maxY = y - 1
} else if minY == 0 && pctColor(img, minColor, y, false) > 0.4 {
minY = y - 1
}
}
if minY < maxY {
return nil
}
return func(y int) float64 {
return math.Round(extrapolate(maxY, minY, maxPrice, minPrice, y)*100) / 100
}
}
// getDateExtrapolator gets an extrapolator to extract the date from the image
// based on an x value relative to the image (NOT the grid).
func getDateExtrapolator(img image.Image, gridBounds image.Rectangle, startDate, endDate time.Time) func(y int) time.Time {
dates := getDaysBetween(startDate, endDate)
return func(y int) time.Time {
return dates[int(math.Round(extrapolate(gridBounds.Min.X+1, gridBounds.Max.X, 0, float64(len(dates)), y)))]
}
}

67
util.go

@ -0,0 +1,67 @@
package main
import (
"image"
"image/color"
"image/png"
"os"
"time"
)
func getDaysBetween(start, end time.Time) []time.Time {
var dates []time.Time
for t := start; t.Before(end); t = t.AddDate(0, 0, 1) {
dates = append(dates, t)
}
return dates
}
func pctColor(img image.Image, fn colorMatcherFunc, p int, isColumn bool) float64 {
var n int
var t int
if isColumn {
t = img.Bounds().Max.Y
} else {
t = img.Bounds().Max.X
}
for op := 0; op < t; op++ {
if (isColumn && fn(img.At(p, op))) || (!isColumn && fn(img.At(op, p))) {
n++
}
}
return float64(n) / float64(t)
}
// colorMatcherFunc returns a bool given a color.
type colorMatcherFunc func(color.Color) bool
// colorMatcher returns a function checking if an color.Color matches a rgb color.
func colorMatcher(r, g, b uint8) colorMatcherFunc {
return func(c color.Color) bool {
cr, cg, cb, _ := c.RGBA()
return uint8(cr>>8) == r && uint8(cg>>8) == g && uint8(cb>>8) == b
}
}
// extrapolate estimates the value of point p on a line given two known points and values.
func extrapolate(p1, p2 int, v1, v2 float64, p int) float64 {
return float64(p-p1)/float64(p2-p1)*float64(v2-v1) + v1
}
// loadPNG loads a PNG image.
func loadPNG(path string) (image.Image, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
return png.Decode(f)
}
// must panics if err is not nil. This is only to simplify testing during development.
func must(v interface{}, err error) interface{} {
if err != nil {
panic(err)
}
return v
}