Fetch updates from scryfall bulk file

* Switch update to bulkupdate
* Remove tmpfile
* Better output for update

pkg/serra/scryfall.go

@@ -5,8 +5,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
+	"os"
 	"time"
 
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -119,18 +121,119 @@ type Card struct {
 	Variation      bool    `json:"variation"`
 }
 
-// Getter for currency specific value
+type BulkIndex struct {
-func (c Card) getValue(foil bool) float64 {
+	Object  string `json:"object"`
-	if getCurrency() == EUR {
+	HasMore bool   `json:"has_more"`
-		if foil {
+	Data    []struct {
-			return c.Prices.EurFoil
+		Object          string    `json:"object"`
+		ID              string    `json:"id"`
+		Type            string    `json:"type"`
+		UpdatedAt       time.Time `json:"updated_at"`
+		URI             string    `json:"uri"`
+		Name            string    `json:"name"`
+		Description     string    `json:"description"`
+		Size            int       `json:"size"`
+		DownloadURI     string    `json:"download_uri"`
+		ContentType     string    `json:"content_type"`
+		ContentEncoding string    `json:"content_encoding"`
+	} `json:"data"`
+}
+
+func fetchBulkDownloadURL() (string, error) {
+	url := "https://api.scryfall.com/bulk-data"
+	downloadURL := ""
+
+	// Make an HTTP GET request
+	resp, err := http.Get(url)
+	if err != nil {
+		log.Fatalf("Error fetching data: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// Read the response body
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatalf("Error reading response body: %v", err)
+	}
+
+	// Unmarshal the JSON response
+	var bulkData BulkIndex
+	if err := json.Unmarshal(body, &bulkData); err != nil {
+		log.Fatalf("Error unmarshaling JSON: %v", err)
+	}
+
+	// Find and print the unique cards URL
+	for _, item := range bulkData.Data {
+		if item.Type == "default_cards" {
+			downloadURL = item.DownloadURI
 		}
-		return c.Prices.Eur
 	}
-	if foil {
+
-		return c.Prices.UsdFoil
+	return downloadURL, nil
+}
+
+func downloadBulkData(downloadURL string) (string, error) {
+
+	// Create a temporary directory
+	tempDir, err := os.MkdirTemp("", "download")
+	if err != nil {
+		log.Fatalf("Error creating temporary directory: %v", err)
 	}
-	return c.Prices.Usd
+	// defer os.RemoveAll(tempDir) // Clean up the directory when done
+
+	// Create a temporary file in the temporary directory
+	tempFile, err := os.CreateTemp(tempDir, "downloaded-*.json") // Adjust the extension if necessary
+	if err != nil {
+		log.Fatalf("Error creating temporary file: %v", err)
+	}
+	// defer tempFile.Close() // Ensure we close the file when we're done
+
+	// Download the file
+	resp, err := http.Get(downloadURL)
+	if err != nil {
+		log.Fatalf("Error downloading file: %v", err)
+	}
+	defer resp.Body.Close() // Make sure to close the response body
+
+	// Check for a successful response
+	if resp.StatusCode != http.StatusOK {
+		log.Fatalf("Error: received status code %d", resp.StatusCode)
+	}
+
+	// Copy the response body to the temporary file
+	_, err = io.Copy(tempFile, resp.Body)
+	if err != nil {
+		log.Fatalf("Error saving file: %v", err)
+	}
+
+	return tempFile.Name(), nil
+}
+
+func loadBulkFile(bulkFilePath string) ([]Card, error) {
+
+	var cards []Card
+	fileBytes, _ := os.ReadFile(bulkFilePath)
+	defer os.Remove(bulkFilePath)
+
+	err := json.Unmarshal(fileBytes, &cards)
+	if err != nil {
+		fmt.Println("Error unmarshalling bulk file:", err)
+		return cards, nil
+	}
+
+	return cards, nil
+
+}
+
+func getCardFromBulk(cards []Card, setName, collectorNumber string) (*Card, error) {
+	var foundCard Card
+	for _, v := range cards {
+		if v.CollectorNumber == collectorNumber && v.Set == setName {
+			foundCard = v
+			return &foundCard, nil
+		}
+	}
+	return &Card{}, fmt.Errorf("Card %s/%s not found in bulk data", setName, collectorNumber)
 }
 
 type PriceEntry struct {
@@ -170,6 +273,20 @@ type Set struct {
 	URI          string             `json:"uri"`
 }
 
+// Getter for currency specific value
+func (c Card) getValue(foil bool) float64 {
+	if getCurrency() == EUR {
+		if foil {
+			return c.Prices.EurFoil
+		}
+		return c.Prices.Eur
+	}
+	if foil {
+		return c.Prices.UsdFoil
+	}
+	return c.Prices.Usd
+}
+
 func fetchCard(setName, collectorNumber string) (*Card, error) {
 	resp, err := http.Get(fmt.Sprintf("https://api.scryfall.com/cards/%s/%s/", setName, collectorNumber))
 	if err != nil {
@@ -181,7 +298,7 @@ func fetchCard(setName, collectorNumber string) (*Card, error) {
 		return &Card{}, fmt.Errorf("Card %s/%s not found", setName, collectorNumber)
 	}
 
-	//We Read the response body on the line below.
+	//we read the response body on the line below.
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		log.Fatalf("%s", err)
@@ -208,7 +325,7 @@ func fetchCard(setName, collectorNumber string) (*Card, error) {
 }
 
 func fetchSets() (*SetList, error) {
-	// TODO better URL Building...
+	// TODO: better URL Building...
 	resp, err := http.Get("https://api.scryfall.com/sets")
 	if err != nil {
 		log.Fatalln(err)

pkg/serra/storage.go

@@ -17,7 +17,8 @@ type Total struct {
 	Value []PriceEntry `bson:"value"`
 }
 
-// https://siongui.github.io/2017/02/11/go-add-method-function-to-type-in-external-package/
+// Collection Struct
+// reason: https://siongui.github.io/2017/02/11/go-add-method-function-to-type-in-external-package/
 type Collection struct {
 	*mongo.Collection
 }

pkg/serra/update.go

@@ -19,8 +19,8 @@ func init() {
 var updateCmd = &cobra.Command{
 	Aliases:       []string{"u"},
 	Use:           "update",
-	Short:         "Update card values from scryfall",
+	Short:         "update card values from scryfall",
-	Long:          `The update mechanism iterates over each card in your collection and fetches its price. After all cards you own in a set are updated, the set value will update. After all Sets are updated, the whole collection value is updated.`,
+	Long:          `the update mechanism iterates over each card in your collection and fetches its price. after all cards you own in a set are updated, the set value will update. after all sets are updated, the whole collection value is updated.`,
 	SilenceErrors: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 
@@ -49,11 +49,32 @@ var updateCmd = &cobra.Command{
 				{"usdfoil", bson.D{{"$sum", bson.D{{"$multiply", bson.A{"$last_price.usd_foil", "$serra_count_foil"}}}}}},
 			}}}
 
+		l.Info("Fetching bulk data from scryfall...")
+		downloadURL, err := fetchBulkDownloadURL()
+		if err != nil {
+			l.Error("Could not extract bulk download URL:", err)
+		}
+		l.Infof("Found latest bulkfile url: %s", downloadURL)
+
+		l.Info("Downloading bulk data file...")
+		bulkFilePath, err := downloadBulkData(downloadURL)
+		if err != nil {
+			l.Error("Could not fetch bulk json from scryfall", err)
+		}
+
+		l.Info("Loading bulk data file...")
+		updatedCards, err := loadBulkFile(bulkFilePath)
+		if err != nil {
+			l.Error("Could not load bulk file:", err)
+		}
+		l.Infof("Successfully loaded %d cards. Starting Update.", len(updatedCards))
+
 		sets, _ := fetchSets()
 		for _, set := range sets.Data {
 
 			// When downloading new sets, PriceList needs to be initialized
 			// This query silently fails if set was already downloaded. Not nice but ok for now.
+			// TODO: make this not fail silently
 			set.SerraPrices = []PriceEntry{}
 			setscoll.storageAddSet(&set)
 
@@ -74,13 +95,13 @@ var updateCmd = &cobra.Command{
 					SaucerHead:    "[green]>[reset]",
 					SaucerPadding: " ",
 					BarStart:      "|",
-					BarEnd:        set.Name,
+					BarEnd:        "| " + set.Name,
 				}),
 			)
 
 			for _, card := range cards {
 				bar.Add(1)
-				updatedCard, err := fetchCard(card.Set, card.CollectorNumber)
+				updatedCard, err := getCardFromBulk(updatedCards, card.Set, card.CollectorNumber)
 				if err != nil {
 					l.Error(err)
 					continue
@@ -115,7 +136,6 @@ var updateCmd = &cobra.Command{
 				"$set":  bson.M{"serra_updated": p.Date, "cardcount": set.CardCount},
 				"$push": bson.M{"serra_prices": p},
 			}
-			// fmt.Printf("Set %s%s%s (%s) is now worth %s%.02f EUR%s\n", Pink, set.Name, Reset, set.Code, Yellow, setvalue[0]["value"], Reset)
 			setscoll.storageUpdate(bson.M{"code": bson.M{"$eq": set.Code}}, setUpdate)
 		}
 
@@ -130,7 +150,7 @@ var updateCmd = &cobra.Command{
 		tmpCard := Card{}
 		tmpCard.Prices = t
 
-		fmt.Printf("\n%sUpdating total value of collection to: %s%.02f%s%s\n", Green, Yellow, tmpCard.getValue(false)+tmpCard.getValue(true), getCurrency(), Reset)
+		l.Info("\n%sUpdating total value of collection to: %s%.02f%s%s\n", Green, Yellow, tmpCard.getValue(false)+tmpCard.getValue(true), getCurrency(), Reset)
 		totalcoll.storageAddTotal(t)
 
 		return nil