diff --git a/pkg/serra/scryfall.go b/pkg/serra/scryfall.go index b08b23e..c5a82e9 100644 --- a/pkg/serra/scryfall.go +++ b/pkg/serra/scryfall.go @@ -5,8 +5,10 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "log" "net/http" + "os" "time" "go.mongodb.org/mongo-driver/bson/primitive" @@ -119,18 +121,119 @@ type Card struct { Variation bool `json:"variation"` } -// Getter for currency specific value -func (c Card) getValue(foil bool) float64 { - if getCurrency() == EUR { - if foil { - return c.Prices.EurFoil +type BulkIndex struct { + Object string `json:"object"` + HasMore bool `json:"has_more"` + Data []struct { + Object string `json:"object"` + ID string `json:"id"` + Type string `json:"type"` + UpdatedAt time.Time `json:"updated_at"` + URI string `json:"uri"` + Name string `json:"name"` + Description string `json:"description"` + Size int `json:"size"` + DownloadURI string `json:"download_uri"` + ContentType string `json:"content_type"` + ContentEncoding string `json:"content_encoding"` + } `json:"data"` +} + +func fetchBulkDownloadURL() (string, error) { + url := "https://api.scryfall.com/bulk-data" + downloadURL := "" + + // Make an HTTP GET request + resp, err := http.Get(url) + if err != nil { + log.Fatalf("Error fetching data: %v", err) + } + defer resp.Body.Close() + + // Read the response body + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Fatalf("Error reading response body: %v", err) + } + + // Unmarshal the JSON response + var bulkData BulkIndex + if err := json.Unmarshal(body, &bulkData); err != nil { + log.Fatalf("Error unmarshaling JSON: %v", err) + } + + // Find and print the unique cards URL + for _, item := range bulkData.Data { + if item.Type == "default_cards" { + downloadURL = item.DownloadURI } - return c.Prices.Eur } - if foil { - return c.Prices.UsdFoil + + return downloadURL, nil +} + +func downloadBulkData(downloadURL string) (string, error) { + + // Create a temporary directory + tempDir, err := os.MkdirTemp("", "download") + if err != nil { + log.Fatalf("Error creating temporary directory: %v", err) } - return c.Prices.Usd + // defer os.RemoveAll(tempDir) // Clean up the directory when done + + // Create a temporary file in the temporary directory + tempFile, err := os.CreateTemp(tempDir, "downloaded-*.json") // Adjust the extension if necessary + if err != nil { + log.Fatalf("Error creating temporary file: %v", err) + } + // defer tempFile.Close() // Ensure we close the file when we're done + + // Download the file + resp, err := http.Get(downloadURL) + if err != nil { + log.Fatalf("Error downloading file: %v", err) + } + defer resp.Body.Close() // Make sure to close the response body + + // Check for a successful response + if resp.StatusCode != http.StatusOK { + log.Fatalf("Error: received status code %d", resp.StatusCode) + } + + // Copy the response body to the temporary file + _, err = io.Copy(tempFile, resp.Body) + if err != nil { + log.Fatalf("Error saving file: %v", err) + } + + return tempFile.Name(), nil +} + +func loadBulkFile(bulkFilePath string) ([]Card, error) { + + var cards []Card + fileBytes, _ := os.ReadFile(bulkFilePath) + defer os.Remove(bulkFilePath) + + err := json.Unmarshal(fileBytes, &cards) + if err != nil { + fmt.Println("Error unmarshalling bulk file:", err) + return cards, nil + } + + return cards, nil + +} + +func getCardFromBulk(cards []Card, setName, collectorNumber string) (*Card, error) { + var foundCard Card + for _, v := range cards { + if v.CollectorNumber == collectorNumber && v.Set == setName { + foundCard = v + return &foundCard, nil + } + } + return &Card{}, fmt.Errorf("Card %s/%s not found in bulk data", setName, collectorNumber) } type PriceEntry struct { @@ -170,6 +273,20 @@ type Set struct { URI string `json:"uri"` } +// Getter for currency specific value +func (c Card) getValue(foil bool) float64 { + if getCurrency() == EUR { + if foil { + return c.Prices.EurFoil + } + return c.Prices.Eur + } + if foil { + return c.Prices.UsdFoil + } + return c.Prices.Usd +} + func fetchCard(setName, collectorNumber string) (*Card, error) { resp, err := http.Get(fmt.Sprintf("https://api.scryfall.com/cards/%s/%s/", setName, collectorNumber)) if err != nil { @@ -181,7 +298,7 @@ func fetchCard(setName, collectorNumber string) (*Card, error) { return &Card{}, fmt.Errorf("Card %s/%s not found", setName, collectorNumber) } - //We Read the response body on the line below. + //we read the response body on the line below. body, err := io.ReadAll(resp.Body) if err != nil { log.Fatalf("%s", err) @@ -208,7 +325,7 @@ func fetchCard(setName, collectorNumber string) (*Card, error) { } func fetchSets() (*SetList, error) { - // TODO better URL Building... + // TODO: better URL Building... resp, err := http.Get("https://api.scryfall.com/sets") if err != nil { log.Fatalln(err) diff --git a/pkg/serra/storage.go b/pkg/serra/storage.go index 0a463b9..d3da367 100644 --- a/pkg/serra/storage.go +++ b/pkg/serra/storage.go @@ -17,7 +17,8 @@ type Total struct { Value []PriceEntry `bson:"value"` } -// https://siongui.github.io/2017/02/11/go-add-method-function-to-type-in-external-package/ +// Collection Struct +// reason: https://siongui.github.io/2017/02/11/go-add-method-function-to-type-in-external-package/ type Collection struct { *mongo.Collection } diff --git a/pkg/serra/update.go b/pkg/serra/update.go index 2b06b63..e811664 100644 --- a/pkg/serra/update.go +++ b/pkg/serra/update.go @@ -19,8 +19,8 @@ func init() { var updateCmd = &cobra.Command{ Aliases: []string{"u"}, Use: "update", - Short: "Update card values from scryfall", - Long: `The update mechanism iterates over each card in your collection and fetches its price. After all cards you own in a set are updated, the set value will update. After all Sets are updated, the whole collection value is updated.`, + Short: "update card values from scryfall", + Long: `the update mechanism iterates over each card in your collection and fetches its price. after all cards you own in a set are updated, the set value will update. after all sets are updated, the whole collection value is updated.`, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -49,11 +49,32 @@ var updateCmd = &cobra.Command{ {"usdfoil", bson.D{{"$sum", bson.D{{"$multiply", bson.A{"$last_price.usd_foil", "$serra_count_foil"}}}}}}, }}} + l.Info("Fetching bulk data from scryfall...") + downloadURL, err := fetchBulkDownloadURL() + if err != nil { + l.Error("Could not extract bulk download URL:", err) + } + l.Infof("Found latest bulkfile url: %s", downloadURL) + + l.Info("Downloading bulk data file...") + bulkFilePath, err := downloadBulkData(downloadURL) + if err != nil { + l.Error("Could not fetch bulk json from scryfall", err) + } + + l.Info("Loading bulk data file...") + updatedCards, err := loadBulkFile(bulkFilePath) + if err != nil { + l.Error("Could not load bulk file:", err) + } + l.Infof("Successfully loaded %d cards. Starting Update.", len(updatedCards)) + sets, _ := fetchSets() for _, set := range sets.Data { // When downloading new sets, PriceList needs to be initialized // This query silently fails if set was already downloaded. Not nice but ok for now. + // TODO: make this not fail silently set.SerraPrices = []PriceEntry{} setscoll.storageAddSet(&set) @@ -74,13 +95,13 @@ var updateCmd = &cobra.Command{ SaucerHead: "[green]>[reset]", SaucerPadding: " ", BarStart: "|", - BarEnd: set.Name, + BarEnd: "| " + set.Name, }), ) for _, card := range cards { bar.Add(1) - updatedCard, err := fetchCard(card.Set, card.CollectorNumber) + updatedCard, err := getCardFromBulk(updatedCards, card.Set, card.CollectorNumber) if err != nil { l.Error(err) continue @@ -115,7 +136,6 @@ var updateCmd = &cobra.Command{ "$set": bson.M{"serra_updated": p.Date, "cardcount": set.CardCount}, "$push": bson.M{"serra_prices": p}, } - // fmt.Printf("Set %s%s%s (%s) is now worth %s%.02f EUR%s\n", Pink, set.Name, Reset, set.Code, Yellow, setvalue[0]["value"], Reset) setscoll.storageUpdate(bson.M{"code": bson.M{"$eq": set.Code}}, setUpdate) } @@ -130,7 +150,7 @@ var updateCmd = &cobra.Command{ tmpCard := Card{} tmpCard.Prices = t - fmt.Printf("\n%sUpdating total value of collection to: %s%.02f%s%s\n", Green, Yellow, tmpCard.getValue(false)+tmpCard.getValue(true), getCurrency(), Reset) + l.Info("\n%sUpdating total value of collection to: %s%.02f%s%s\n", Green, Yellow, tmpCard.getValue(false)+tmpCard.getValue(true), getCurrency(), Reset) totalcoll.storageAddTotal(t) return nil