shylie/serra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fetch updates from scryfall bulk file

Browse Source 
* Switch update to bulkupdate
* Remove tmpfile
* Better output for update
This commit is contained in:
Florian Baumann 2025-12-01 09:31:52 +01:00
parent a9d1fbc2cd
commit 507eef148a
3 changed files with 156 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
pkg/serra/scryfall.go
View File

@ -5,8 +5,10 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"time"
"go.mongodb.org/mongo-driver/bson/primitive"
@ -119,18 +121,119 @@ type Card struct {
Variation bool `json:"variation"`
}
// Getter for currency specific value
func (c Card) getValue(foil bool) float64 {
if getCurrency() == EUR {
if foil {
return c.Prices.EurFoil
type BulkIndex struct {
Object string `json:"object"`
HasMore bool `json:"has_more"`
Data []struct {
Object string `json:"object"`
ID string `json:"id"`
Type string `json:"type"`
UpdatedAt time.Time `json:"updated_at"`
URI string `json:"uri"`
Name string `json:"name"`
Description string `json:"description"`
Size int `json:"size"`
DownloadURI string `json:"download_uri"`
ContentType string `json:"content_type"`
ContentEncoding string `json:"content_encoding"`
} `json:"data"`
}
func fetchBulkDownloadURL() (string, error) {
url := "https://api.scryfall.com/bulk-data"
downloadURL := ""
// Make an HTTP GET request
resp, err := http.Get(url)
if err != nil {
log.Fatalf("Error fetching data: %v", err)
}
defer resp.Body.Close()
// Read the response body
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalf("Error reading response body: %v", err)
}
// Unmarshal the JSON response
var bulkData BulkIndex
if err := json.Unmarshal(body, &bulkData); err != nil {
log.Fatalf("Error unmarshaling JSON: %v", err)
}
// Find and print the unique cards URL
for _, item := range bulkData.Data {
if item.Type == "default_cards" {
downloadURL = item.DownloadURI
}
return c.Prices.Eur
}
if foil {
return c.Prices.UsdFoil
return downloadURL, nil
}
func downloadBulkData(downloadURL string) (string, error) {
// Create a temporary directory
tempDir, err := os.MkdirTemp("", "download")
if err != nil {
log.Fatalf("Error creating temporary directory: %v", err)
}
return c.Prices.Usd
// defer os.RemoveAll(tempDir) // Clean up the directory when done
// Create a temporary file in the temporary directory
tempFile, err := os.CreateTemp(tempDir, "downloaded-*.json") // Adjust the extension if necessary
if err != nil {
log.Fatalf("Error creating temporary file: %v", err)
}
// defer tempFile.Close() // Ensure we close the file when we're done
// Download the file
resp, err := http.Get(downloadURL)
if err != nil {
log.Fatalf("Error downloading file: %v", err)
}
defer resp.Body.Close() // Make sure to close the response body
// Check for a successful response
if resp.StatusCode != http.StatusOK {
log.Fatalf("Error: received status code %d", resp.StatusCode)
}
// Copy the response body to the temporary file
_, err = io.Copy(tempFile, resp.Body)
if err != nil {
log.Fatalf("Error saving file: %v", err)
}
return tempFile.Name(), nil
}
func loadBulkFile(bulkFilePath string) ([]Card, error) {
var cards []Card
fileBytes, _ := os.ReadFile(bulkFilePath)
defer os.Remove(bulkFilePath)
err := json.Unmarshal(fileBytes, &cards)
if err != nil {
fmt.Println("Error unmarshalling bulk file:", err)
return cards, nil
}
return cards, nil
}
func getCardFromBulk(cards []Card, setName, collectorNumber string) (*Card, error) {
var foundCard Card
for _, v := range cards {
if v.CollectorNumber == collectorNumber && v.Set == setName {
foundCard = v
return &foundCard, nil
}
}
return &Card{}, fmt.Errorf("Card %s/%s not found in bulk data", setName, collectorNumber)
}
type PriceEntry struct {
@ -170,6 +273,20 @@ type Set struct {
URI string `json:"uri"`
}
// Getter for currency specific value
func (c Card) getValue(foil bool) float64 {
if getCurrency() == EUR {
if foil {
return c.Prices.EurFoil
}
return c.Prices.Eur
}
if foil {
return c.Prices.UsdFoil
}
return c.Prices.Usd
}
func fetchCard(setName, collectorNumber string) (*Card, error) {
resp, err := http.Get(fmt.Sprintf("https://api.scryfall.com/cards/%s/%s/", setName, collectorNumber))
if err != nil {
@ -181,7 +298,7 @@ func fetchCard(setName, collectorNumber string) (*Card, error) {
return &Card{}, fmt.Errorf("Card %s/%s not found", setName, collectorNumber)
}
//We Read the response body on the line below.
//we read the response body on the line below.
body, err := io.ReadAll(resp.Body)
if err != nil {
log.Fatalf("%s", err)
@ -208,7 +325,7 @@ func fetchCard(setName, collectorNumber string) (*Card, error) {
}
func fetchSets() (*SetList, error) {
// TODO better URL Building...
// TODO: better URL Building...
resp, err := http.Get("https://api.scryfall.com/sets")
if err != nil {
log.Fatalln(err)

3
pkg/serra/storage.go
View File

@ -17,7 +17,8 @@ type Total struct {
Value []PriceEntry `bson:"value"`
}
// https://siongui.github.io/2017/02/11/go-add-method-function-to-type-in-external-package/
// Collection Struct
// reason: https://siongui.github.io/2017/02/11/go-add-method-function-to-type-in-external-package/
type Collection struct {
*mongo.Collection
}

32
pkg/serra/update.go
View File

@ -19,8 +19,8 @@ func init() {
var updateCmd = &cobra.Command{
Aliases: []string{"u"},
Use: "update",
Short: "Update card values from scryfall",
Long: `The update mechanism iterates over each card in your collection and fetches its price. After all cards you own in a set are updated, the set value will update. After all Sets are updated, the whole collection value is updated.`,
Short: "update card values from scryfall",
Long: `the update mechanism iterates over each card in your collection and fetches its price. after all cards you own in a set are updated, the set value will update. after all sets are updated, the whole collection value is updated.`,
SilenceErrors: true,
RunE: func(cmd *cobra.Command, args []string) error {
@ -49,11 +49,32 @@ var updateCmd = &cobra.Command{
{"usdfoil", bson.D{{"$sum", bson.D{{"$multiply", bson.A{"$last_price.usd_foil", "$serra_count_foil"}}}}}},
}}}
l.Info("Fetching bulk data from scryfall...")
downloadURL, err := fetchBulkDownloadURL()
if err != nil {
l.Error("Could not extract bulk download URL:", err)
}
l.Infof("Found latest bulkfile url: %s", downloadURL)
l.Info("Downloading bulk data file...")
bulkFilePath, err := downloadBulkData(downloadURL)
if err != nil {
l.Error("Could not fetch bulk json from scryfall", err)
}
l.Info("Loading bulk data file...")
updatedCards, err := loadBulkFile(bulkFilePath)
if err != nil {
l.Error("Could not load bulk file:", err)
}
l.Infof("Successfully loaded %d cards. Starting Update.", len(updatedCards))
sets, _ := fetchSets()
for _, set := range sets.Data {
// When downloading new sets, PriceList needs to be initialized
// This query silently fails if set was already downloaded. Not nice but ok for now.
// TODO: make this not fail silently
set.SerraPrices = []PriceEntry{}
setscoll.storageAddSet(&set)
@ -74,13 +95,13 @@ var updateCmd = &cobra.Command{
SaucerHead: "[green]>[reset]",
SaucerPadding: " ",
BarStart: "|",
BarEnd: set.Name,
BarEnd: "| " + set.Name,
}),
)
for _, card := range cards {
bar.Add(1)
updatedCard, err := fetchCard(card.Set, card.CollectorNumber)
updatedCard, err := getCardFromBulk(updatedCards, card.Set, card.CollectorNumber)
if err != nil {
l.Error(err)
continue
@ -115,7 +136,6 @@ var updateCmd = &cobra.Command{
"$set": bson.M{"serra_updated": p.Date, "cardcount": set.CardCount},
"$push": bson.M{"serra_prices": p},
}
// fmt.Printf("Set %s%s%s (%s) is now worth %s%.02f EUR%s\n", Pink, set.Name, Reset, set.Code, Yellow, setvalue[0]["value"], Reset)
setscoll.storageUpdate(bson.M{"code": bson.M{"$eq": set.Code}}, setUpdate)
}
@ -130,7 +150,7 @@ var updateCmd = &cobra.Command{
tmpCard := Card{}
tmpCard.Prices = t
fmt.Printf("\n%sUpdating total value of collection to: %s%.02f%s%s\n", Green, Yellow, tmpCard.getValue(false)+tmpCard.getValue(true), getCurrency(), Reset)
l.Info("\n%sUpdating total value of collection to: %s%.02f%s%s\n", Green, Yellow, tmpCard.getValue(false)+tmpCard.getValue(true), getCurrency(), Reset)
totalcoll.storageAddTotal(t)
return nil