Browse Source

Command line tool to get and parse data

master
Patrick Gaskin 1 year ago
parent
commit
60524dbfda
Signed by: geek1011 GPG Key ID: A2FD79F68A2AB707
7 changed files with 209 additions and 5 deletions
  1. +1
    -0
      .gitignore
  2. +4
    -0
      README.md
  3. +121
    -0
      camelcamelcamel/camel.go
  4. +2
    -0
      go.mod
  5. +2
    -0
      go.sum
  6. +5
    -5
      graph/graph.go
  7. +74
    -0
      main.go

+ 1
- 0
.gitignore View File

@ -0,0 +1 @@
*.csv

+ 4
- 0
README.md View File

@ -0,0 +1,4 @@
# camelparse
camelparse parses historical Amazon price data from camelcamelcamel. It parses
the price graph images quite accurately (no issues when tested with ~4 years of
data from B01N5IB20Q and B0167AK9Y8).

+ 121
- 0
camelcamelcamel/camel.go View File

@ -0,0 +1,121 @@
package camelcamelcamel
import (
"encoding/json"
"fmt"
"image"
"image/png"
"io/ioutil"
"net/http"
"strings"
"time"
)
// UserAgent is used for API calls. It should be a Chrome one, as the API is the one from the Chrome extension.
var UserAgent = `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36`
// CamelizerResponse is a response from the camelcamelcamel API.
type CamelizerResponse struct {
ASIN string `json:"asin"`
Title string `json:"title"`
CurrencySymbol string `json:"currency_symbol"`
CreatedAt *time.Time `json:"created_at"`
Prices struct {
PriceAmazon *CamelPrice `json:"price_amazon"`
PriceNew *CamelPrice `json:"price_new"`
PriceUsed *CamelPrice `json:"price_used"`
} `json:"prices"`
LastPrice struct {
PriceAmazon *CamelPrice `json:"price_amazon"`
PriceNew *CamelPrice `json:"price_new"`
PriceUsed *CamelPrice `json:"price_used"`
} `json:"last_price"`
HighestPricing struct {
PriceAmazon *CamelTimedPrice `json:"price_amazon"`
PriceNew *CamelTimedPrice `json:"price_new"`
PriceUsed *CamelTimedPrice `json:"price_used"`
} `json:"highest_pricing"`
LowestPricing struct {
PriceAmazon *CamelTimedPrice `json:"price_amazon"`
PriceNew *CamelTimedPrice `json:"price_new"`
PriceUsed *CamelTimedPrice `json:"price_used"`
} `json:"lowest_pricing"`
}
// CamelTimedPrice is a price with a timestamp.
type CamelTimedPrice struct {
CreatedAt CamelDate `json:"created_at"`
Price CamelPrice `json:"price"`
}
// CamelPrice is the camelcamelcamel price format (in cents).
type CamelPrice int
// Price returns the actual price in dollars.
func (p CamelPrice) Price() float64 {
return float64(p) / 100
}
// CamelDate is the camelcamelcamel date format.
type CamelDate string
// Date returns the parsed time or panics.
func (d CamelDate) Date() time.Time {
t, err := time.Parse("Jan 2, 2006", string(d))
if err != nil {
panic(err)
}
return t
}
// Camelizer calls the camelcamelcamel API.
func Camelizer(asin, locale string) (*CamelizerResponse, error) {
req, err := http.NewRequest("GET", fmt.Sprintf("https://camelcamelcamel.com/chromelizer/%s?locale=%s&ver=5", asin, strings.ToUpper(locale)), nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", UserAgent)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status %s, api response %#v", resp.Status, string(buf))
}
var cr CamelizerResponse
if err := json.Unmarshal(buf, &cr); err != nil {
return nil, err
}
if strings.ToUpper(cr.ASIN) != strings.ToUpper(asin) {
return nil, fmt.Errorf("api error, expected asin %#v, got %#v", asin, cr.ASIN)
}
return &cr, nil
}
// Graph returns the price graph for the Amazon seller of an ASIN.
func Graph(asin, locale string, width, height int) (image.Image, error) {
req, err := http.NewRequest("GET", fmt.Sprintf("https://charts.camelcamelcamel.com/%s/%s/amazon.png?force=1&zero=0&w=%d&h=%d&desired=false&legend=0&ilt=1&tp=all&fo=0&lang=en", strings.ToLower(locale), asin, width, height), nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", UserAgent)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return png.Decode(resp.Body)
}

+ 2
- 0
go.mod View File

@ -1,3 +1,5 @@
module git.geek1011.net/geek1011/camelparse
go 1.12
require github.com/spf13/pflag v1.0.3

+ 2
- 0
go.sum View File

@ -0,0 +1,2 @@
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=

+ 5
- 5
graph/graph.go View File

@ -103,7 +103,7 @@ func traceLine(img image.Image, graphBounds image.Rectangle, line colorMatcherFu
}
if curY == -1 {
if x < graphBounds.Max.X {
return fmt.Errorf("line ended early at %d", x)
return fmt.Errorf("could not find line at %d (maybe the graph is too small?)", x)
}
break
}
@ -113,17 +113,17 @@ func traceLine(img image.Image, graphBounds image.Rectangle, line colorMatcherFu
}
// getGraphBounds returns the detected graph bounds (based on lines of the grid
// color at least 75% of the image width).
// color at least 75% of the image width, 15% for horizontal as dashed may overlap).
func getGraphBounds(img image.Image, fn colorMatcherFunc) *image.Rectangle {
var yTop, yBottom, xLeft, xRight int
for y := 0; y < img.Bounds().Max.Y; y++ {
if pctColor(img, fn, y, false) > 0.75 {
yTop = y
if pctColor(img, fn, y, false) > 0.15 {
yTop = y - 1 // line is bottom of actual 2px line
break
}
}
for y := img.Bounds().Max.Y; y > 0; y-- {
if pctColor(img, fn, y, false) > 0.75 {
if pctColor(img, fn, y, false) > 0.15 {
yBottom = y
break
}

+ 74
- 0
main.go View File

@ -0,0 +1,74 @@
package main
import (
"fmt"
"image/png"
"os"
"time"
"git.geek1011.net/geek1011/camelparse/camelcamelcamel"
"git.geek1011.net/geek1011/camelparse/graph"
"github.com/spf13/pflag"
)
func main() {
locale := pflag.StringP("locale", "l", "us", "Amazon store location (gb, us, ca)")
writeGraph := pflag.StringP("write-graph", "g", "", "Write the PNG graph to a file (overwrites if exists)")
csv := pflag.BoolP("csv", "c", false, "CSV output (date,price,outofstock)")
help := pflag.Bool("help", false, "Show this help text")
pflag.Parse()
if *help || pflag.NArg() != 1 {
fmt.Fprintf(os.Stderr, "Usage: %s ASIN\n\nOptions:\n", os.Args[0])
pflag.PrintDefaults()
fmt.Fprintf(os.Stderr, "\nNote: Only items sold by Amazon are currently supported\n")
os.Exit(2)
}
if *locale != "gb" && *locale != "us" && *locale != "ca" {
fmt.Fprintf(os.Stderr, "Error: Unsupported locale %#v, see --help for supported options.", locale)
os.Exit(1)
}
asin := pflag.Arg(0)
cr, err := camelcamelcamel.Camelizer(asin, *locale)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: Could not camelize: %v\n", err)
os.Exit(1)
}
cg, err := camelcamelcamel.Graph(asin, *locale, 16000, 8000)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: Could not get graph: %v\n", err)
os.Exit(1)
}
if *writeGraph != "" {
f, err := os.Create(*writeGraph)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: Could not create graph file: %v\n", err)
os.Exit(1)
}
if err := png.Encode(f, cg); err != nil {
fmt.Fprintf(os.Stderr, "Error: Could not write graph file: %v\n", err)
f.Close()
os.Exit(1)
}
f.Close()
}
pc, err := graph.Parse(cg, cr.HighestPricing.PriceAmazon.Price.Price(), cr.LowestPricing.PriceAmazon.Price.Price(), *cr.CreatedAt, time.Now())
if err != nil {
fmt.Fprintf(os.Stderr, "Error: Could not parse graph file: %v\n", err)
os.Exit(1)
}
for _, p := range pc {
if *csv {
fmt.Println(p.CSV())
} else {
fmt.Println(p)
}
}
}