diff --git a/api/server.go b/api/server.go
index 11edccf1..61b13660 100644
--- a/api/server.go
+++ b/api/server.go
@@ -170,6 +170,15 @@ func NewApiServer(config config.Config) *ApiServer {
panic(err)
}
+ // Entries carry their own freshness window so expired sitemap pages can be
+ // served stale while a background refresh rebuilds them.
+ sitemapPageCache, err := otter.MustBuilder[string, sitemapPageCacheEntry](256).
+ CollectStats().
+ Build()
+ if err != nil {
+ panic(err)
+ }
+
privateKey, err := crypto.HexToECDSA(config.DelegatePrivateKey)
if err != nil {
panic(err)
@@ -281,6 +290,7 @@ func NewApiServer(config config.Config) *ApiServer {
qualifiedPlaylistsCache: &qualifiedPlaylistsCache,
relatedUsersCache: &relatedUsersCache,
genresPopularCache: &genresPopularCache,
+ sitemapPageCache: &sitemapPageCache,
requestValidator: requestValidator,
rewardAttester: rewardAttester,
transactionSender: transactionSender,
@@ -841,6 +851,7 @@ type ApiServer struct {
qualifiedPlaylistsCache *otter.Cache[string, []int32]
relatedUsersCache *otter.Cache[string, []int32]
genresPopularCache *otter.Cache[string, []PopularGenre]
+ sitemapPageCache *otter.Cache[string, sitemapPageCacheEntry]
requestValidator *RequestValidator
rewardManagerClient *reward_manager.RewardManagerClient
claimableTokensClient *claimable_tokens.ClaimableTokensClient
diff --git a/api/v1_sitemaps.go b/api/v1_sitemaps.go
index 0ec6f697..09ffca98 100644
--- a/api/v1_sitemaps.go
+++ b/api/v1_sitemaps.go
@@ -18,6 +18,7 @@ import (
const sitemapLimit = 40_000
const sitemapCountCacheTTL = 1 * time.Hour
+const sitemapPageCacheTTL = 1 * time.Hour
var sitemapPageRegex = regexp.MustCompile(`^(\d+)\.xml$`)
@@ -55,6 +56,12 @@ type cachedCount struct {
refreshing bool // true if a background refresh is already in flight
}
+type sitemapPageCacheEntry struct {
+ data []byte
+ expiresAt time.Time
+ refreshing bool
+}
+
var (
sitemapCountCache = make(map[string]cachedCount)
sitemapCountMu sync.Mutex
@@ -87,6 +94,44 @@ func setCachedCount(key string, value int64) {
}
}
+func (app *ApiServer) getCachedSitemapPage(key string) (data []byte, exists bool, needsRefresh bool) {
+ if app.sitemapPageCache == nil {
+ return nil, false, false
+ }
+ entry, ok := app.sitemapPageCache.Get(key)
+ if !ok {
+ return nil, false, false
+ }
+ if time.Now().After(entry.expiresAt) && !entry.refreshing {
+ entry.refreshing = true
+ app.sitemapPageCache.Set(key, entry)
+ return entry.data, true, true
+ }
+ return entry.data, true, false
+}
+
+func (app *ApiServer) setCachedSitemapPage(key string, data []byte) {
+ if app.sitemapPageCache == nil {
+ return
+ }
+ app.sitemapPageCache.Set(key, sitemapPageCacheEntry{
+ data: data,
+ expiresAt: time.Now().Add(sitemapPageCacheTTL),
+ })
+}
+
+func (app *ApiServer) clearSitemapPageRefreshing(key string) {
+ if app.sitemapPageCache == nil {
+ return
+ }
+ entry, ok := app.sitemapPageCache.Get(key)
+ if !ok {
+ return
+ }
+ entry.refreshing = false
+ app.sitemapPageCache.Set(key, entry)
+}
+
type sitemapURL struct {
XMLName xml.Name `xml:"url"`
Loc string `xml:"loc"`
@@ -266,8 +311,42 @@ func (app *ApiServer) sitemapTypePage(c *fiber.Ctx) error {
return fiber.NewError(400, "Page number must be >= 1")
}
+ cacheKey := entityType + ":" + fileName
+ if data, ok, needsRefresh := app.getCachedSitemapPage(cacheKey); ok {
+ if needsRefresh {
+ go app.refreshSitemapPage(cacheKey, entityType, pageNumber)
+ }
+ c.Set("Content-Type", "text/xml")
+ return c.Send(data)
+ }
+
+ data, err := app.buildSitemapPage(c.Context(), entityType, pageNumber)
+ if err != nil {
+ return err
+ }
+ app.setCachedSitemapPage(cacheKey, data)
+ c.Set("Content-Type", "text/xml")
+ return c.Send(data)
+}
+
+func (app *ApiServer) refreshSitemapPage(cacheKey string, entityType string, pageNumber int) {
+ data, err := app.buildSitemapPage(context.Background(), entityType, pageNumber)
+ if err != nil {
+ app.clearSitemapPageRefreshing(cacheKey)
+ app.logger.Error(
+ "failed to refresh sitemap page",
+ zap.String("cache_key", cacheKey),
+ zap.String("type", entityType),
+ zap.Int("page", pageNumber),
+ zap.Error(err),
+ )
+ return
+ }
+ app.setCachedSitemapPage(cacheKey, data)
+}
+
+func (app *ApiServer) buildSitemapPage(ctx context.Context, entityType string, pageNumber int) ([]byte, error) {
offset := int32((pageNumber - 1) * sitemapLimit)
- ctx := c.Context()
baseURL := app.audiusAppUrl
var slugs []string
@@ -280,7 +359,7 @@ func (app *ApiServer) sitemapTypePage(c *fiber.Ctx) error {
Offset: offset,
})
if e != nil {
- return e
+ return nil, e
}
for _, r := range rows {
if r.Handle.Valid {
@@ -294,7 +373,7 @@ func (app *ApiServer) sitemapTypePage(c *fiber.Ctx) error {
Offset: offset,
})
if e != nil {
- return e
+ return nil, e
}
for _, r := range rows {
if r.Handle.Valid {
@@ -312,7 +391,7 @@ func (app *ApiServer) sitemapTypePage(c *fiber.Ctx) error {
Offset: offset,
})
if e != nil {
- return e
+ return nil, e
}
for _, h := range handles {
if h.Valid {
@@ -321,13 +400,12 @@ func (app *ApiServer) sitemapTypePage(c *fiber.Ctx) error {
}
default:
- return fiber.NewError(400, fmt.Sprintf("Invalid sitemap type %s, should be one of track, playlist, user", entityType))
+ return nil, fiber.NewError(400, fmt.Sprintf("Invalid sitemap type %s, should be one of track, playlist, user", entityType))
}
data, err := buildURLSet(slugs, baseURL)
if err != nil {
- return err
+ return nil, err
}
- c.Set("Content-Type", "text/xml")
- return c.Send(data)
+ return data, nil
}
diff --git a/api/v1_sitemaps_test.go b/api/v1_sitemaps_test.go
index c7a27f51..030b4e52 100644
--- a/api/v1_sitemaps_test.go
+++ b/api/v1_sitemaps_test.go
@@ -3,6 +3,7 @@ package api
import (
"strings"
"testing"
+ "time"
"api.audius.co/database"
"github.com/stretchr/testify/assert"
@@ -112,6 +113,40 @@ func TestSitemapTrackPage(t *testing.T) {
assert.NotContains(t, xml, "%2F")
}
+func TestSitemapTrackPageCache(t *testing.T) {
+ app := sitemapTestApp(t)
+
+ cached := []byte(`cached-track-page`)
+ app.setCachedSitemapPage("track:1.xml", cached)
+
+ status, body := testGet(t, app, "/sitemaps/track/1.xml")
+ require.Equal(t, 200, status)
+ assert.Contains(t, string(body), "cached-track-page")
+}
+
+func TestSitemapTrackPageStaleCacheRefreshesInBackground(t *testing.T) {
+ app := sitemapTestApp(t)
+
+ stale := []byte(`stale-track-page`)
+ require.True(t, app.sitemapPageCache.Set("track:1.xml", sitemapPageCacheEntry{
+ data: stale,
+ expiresAt: time.Now().Add(-time.Minute),
+ }))
+
+ status, body := testGet(t, app, "/sitemaps/track/1.xml")
+ require.Equal(t, 200, status)
+ assert.Contains(t, string(body), "stale-track-page")
+
+ require.Eventually(t, func() bool {
+ entry, ok := app.sitemapPageCache.Get("track:1.xml")
+ return ok &&
+ !entry.refreshing &&
+ time.Now().Before(entry.expiresAt) &&
+ strings.Contains(string(entry.data), "artist1/listed-track") &&
+ !strings.Contains(string(entry.data), "stale-track-page")
+ }, time.Second, 10*time.Millisecond)
+}
+
func TestSitemapPlaylistPage(t *testing.T) {
app := sitemapTestApp(t)
status, body := testGet(t, app, "/sitemaps/playlist/1.xml")