From 1119ca78756c26bc7d2526f01034c599e989f559 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 1 Jul 2026 19:17:05 +0200 Subject: [PATCH 1/3] use self-hosted forest for test RPC --- service/epochutil/epoch_test.go | 2 +- util/testutil/calibnet.go | 5 ++++- util/testutil/testutils.go | 16 +++++++++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/service/epochutil/epoch_test.go b/service/epochutil/epoch_test.go index 65bec75fa..dedf7b683 100644 --- a/service/epochutil/epoch_test.go +++ b/service/epochutil/epoch_test.go @@ -16,7 +16,7 @@ func TestDefaultValue(t *testing.T) { func TestCalibNet(t *testing.T) { // This test may fail when calibnet resets - err := Initialize(context.Background(), "https://api.calibration.node.glif.io/rpc/v1", "") + err := Initialize(context.Background(), testutil.CalibnetRPC, "") require.NoError(t, err) require.EqualValues(t, 1667326380, GenesisTimestamp) } diff --git a/util/testutil/calibnet.go b/util/testutil/calibnet.go index d95d12f50..a389a8bf1 100644 --- a/util/testutil/calibnet.go +++ b/util/testutil/calibnet.go @@ -1,8 +1,11 @@ package testutil // calibnet constants shared by integration tests + +// CalibnetRPC defaults to the self-hosted forest gateway; glif rate-limited CI. +var CalibnetRPC = forestRPC("SINGULARITY_TEST_CALIBNET_RPC", "https://static.187.115.235.167.clients.your-server.de/calibnet/rpc/v1") + const ( - CalibnetRPC = "https://api.calibration.node.glif.io/rpc/v1" CalibnetChainID = 314159 CalibnetDDOContract = "0x889fD50196BE300D06dc4b8F0F17fdB0af587095" CalibnetPaymentsContract = "0x09a0fDc2723fAd1A7b8e3e00eE5DF73841df55a0" diff --git a/util/testutil/testutils.go b/util/testutil/testutils.go index 8955e69a3..ee0fd0fb6 100644 --- a/util/testutil/testutils.go +++ b/util/testutil/testutils.go @@ -55,9 +55,19 @@ const TestWalletAddr = "f1fib3pv7jua2ockdugtz7viz3cyy6lkhh7rfx3sa" const TestPrivateKeyHex = "7b2254797065223a22736563703235366b31222c22507269766174654b6579223a226b35507976337148327349586343595a58594f5775453149326e32554539436861556b6c4e36695a5763453d227d" -// TestLotusAPI is the Lotus API endpoint to use for tests -// Using /rpc/v1 (stable) instead of deprecated /rpc/v0 -const TestLotusAPI = "https://api.node.glif.io/rpc/v1" +// TestLotusAPI defaults to the self-hosted forest gateway; glif rate-limited CI. +var TestLotusAPI = forestRPC("SINGULARITY_TEST_LOTUS_API", "https://static.187.115.235.167.clients.your-server.de/rpc/v1") + +func forestRPC(override, base string) string { + if v := os.Getenv(override); v != "" { + return v + } + // gate rejects untokened requests; carry FOREST_JWT as a query param + if tok := os.Getenv("FOREST_JWT"); tok != "" { + return base + "?access_token=" + tok + } + return base +} // SkipIfNotExternalAPI skips the test if SINGULARITY_TEST_EXTERNAL_API is not set // Use this for tests that make external API calls (e.g., Lotus/Filecoin APIs) From 63da257128238a90305d04cd5360ab732de5be7b Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Mon, 13 Jul 2026 14:10:10 +0200 Subject: [PATCH 2/3] fully wire forest as RPC provider --- .devcontainer/devcontainer.json | 4 ++- .github/workflows/devcontainer-podman.yml | 9 ++++++- service/dealpusher/dealpusher_test.go | 18 ++++++------- service/dealtracker/ddo_tracking_test.go | 2 +- service/dealtracker/dealtracker_test.go | 4 +-- service/epochutil/epoch_test.go | 10 ++++---- util/testutil/anvil.go | 11 ++++++-- util/testutil/calibnet.go | 2 +- util/testutil/testutils.go | 31 +++++++++++++++-------- util/util_test.go | 19 ++++---------- 10 files changed, 63 insertions(+), 47 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 984c7fc50..3fef76939 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -39,7 +39,9 @@ "GOTMPDIR": "/home/vscode/.local/share/tmp", "PGDATA": "/home/vscode/.local/share/pg/pgdata", "PGPORT": "55432", - "PGSOCK_DIR": "/home/vscode/.local/share/pg" + "PGSOCK_DIR": "/home/vscode/.local/share/pg", + "FOREST_JWT": "${localEnv:FOREST_JWT}", + "SINGULARITY_TEST_EXTERNAL_API": "${localEnv:SINGULARITY_TEST_EXTERNAL_API}" }, "runArgs": ["--userns=keep-id"], "containerUser": "vscode" diff --git a/.github/workflows/devcontainer-podman.yml b/.github/workflows/devcontainer-podman.yml index f70f3a99d..e3e1b8152 100644 --- a/.github/workflows/devcontainer-podman.yml +++ b/.github/workflows/devcontainer-podman.yml @@ -54,6 +54,9 @@ jobs: disable-docker: true cache-storage: true + # containerEnv in devcontainer.json pulls these via ${localEnv:...} at + # container start; scope the secret to this step only. FOREST_JWT is + # empty on fork PRs (GitHub policy); SkipIfNotExternalAPI catches that. - name: Build devcontainer id: build uses: parkan/github-actions/devcontainer-build@v3 @@ -61,6 +64,9 @@ jobs: workspace-folder: . container-runtime: podman container-id-label: ci=podman + env: + FOREST_JWT: ${{ secrets.FOREST_JWT }} + SINGULARITY_TEST_EXTERNAL_API: 'true' - name: Generate swagger code if: steps.codegen.outputs.needed == 'true' @@ -103,7 +109,8 @@ jobs: with: container-id: ${{ steps.build.outputs.container-id }} command: > - cd /workspaces/singularity && mkdir -p artifacts && make test GOTESTSUM_ARGS="--junitfile artifacts/unit-tests.xml" + cd /workspaces/singularity && mkdir -p artifacts && + make test GOTESTSUM_ARGS="--junitfile artifacts/unit-tests.xml" container-runtime: podman - name: Run integration tests diff --git a/service/dealpusher/dealpusher_test.go b/service/dealpusher/dealpusher_test.go index 210871359..de7bdb3eb 100644 --- a/service/dealpusher/dealpusher_test.go +++ b/service/dealpusher/dealpusher_test.go @@ -126,7 +126,7 @@ func (m *MockDealMaker) MakeDeal(ctx context.Context, actorObj model.Actor, car func TestDealMakerService_Start(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) ctx, cancel := context.WithCancel(ctx) exitErr := make(chan error, 1) @@ -140,9 +140,9 @@ func TestDealMakerService_Start(t *testing.T) { func TestDealMakerService_MultipleInstances(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service1, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service1, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) - service2, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service2, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) ctx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() @@ -161,7 +161,7 @@ func TestDealMakerService_FailtoSend(t *testing.T) { waitPendingInterval = time.Minute }() testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 2, 0) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 2, 0) require.NoError(t, err) mockDealmaker := new(MockDealMaker) service.dealMaker = mockDealmaker @@ -215,7 +215,7 @@ func TestDealMakerService_Cron(t *testing.T) { waitPendingInterval = time.Minute }() testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) mockDealmaker := new(MockDealMaker) service.dealMaker = mockDealmaker @@ -312,7 +312,7 @@ func TestDealMakerService_ScheduleWithConstraints(t *testing.T) { waitPendingInterval = time.Minute }() testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) mockDealmaker := new(MockDealMaker) service.dealMaker = mockDealmaker @@ -421,7 +421,7 @@ func TestDealMakerService_ScheduleWithConstraints(t *testing.T) { func TestDealmakerService_Force(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) mockDealmaker := new(MockDealMaker) service.dealMaker = mockDealmaker @@ -480,7 +480,7 @@ func TestDealmakerService_Force(t *testing.T) { func TestDealMakerService_MaxReplica(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 1) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 1) require.NoError(t, err) mockDealmaker := new(MockDealMaker) service.dealMaker = mockDealmaker @@ -537,7 +537,7 @@ func TestDealMakerService_MaxReplica(t *testing.T) { func TestDealMakerService_NewScheduleOneOff(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) mockDealmaker := new(MockDealMaker) service.dealMaker = mockDealmaker diff --git a/service/dealtracker/ddo_tracking_test.go b/service/dealtracker/ddo_tracking_test.go index f14d31b8a..d99a23606 100644 --- a/service/dealtracker/ddo_tracking_test.go +++ b/service/dealtracker/ddo_tracking_test.go @@ -202,7 +202,7 @@ func TestRunOnce_DoesNotEpochExpireDDODeals(t *testing.T) { url, server := setupTestServerWithBody(t, `{}`) defer server.Close() - tracker := NewDealTracker(db, 0, url, testutil.TestLotusAPI, "", true) + tracker := NewDealTracker(db, 0, url, testutil.TestLotusAPI, testutil.TestLotusToken, true) require.NoError(t, tracker.runOnce(ctx)) var ddoActive model.Deal diff --git a/service/dealtracker/dealtracker_test.go b/service/dealtracker/dealtracker_test.go index b36b9c244..7e23699c9 100644 --- a/service/dealtracker/dealtracker_test.go +++ b/service/dealtracker/dealtracker_test.go @@ -338,7 +338,7 @@ func TestRunOnce(t *testing.T) { url, server := setupTestServerWithBody(t, string(body)) defer server.Close() require.NoError(t, err) - tracker := NewDealTracker(db, time.Minute, url, testutil.TestLotusAPI, "", true) + tracker := NewDealTracker(db, time.Minute, url, testutil.TestLotusAPI, testutil.TestLotusToken, true) err = tracker.runOnce(context.Background()) require.NoError(t, err) var allDeals []model.Deal @@ -403,7 +403,7 @@ func TestRunOnce_DoesNotEpochExpirePDPDeals(t *testing.T) { url, server := setupTestServerWithBody(t, `{}`) defer server.Close() - tracker := NewDealTracker(db, time.Minute, url, testutil.TestLotusAPI, "", true) + tracker := NewDealTracker(db, time.Minute, url, testutil.TestLotusAPI, testutil.TestLotusToken, true) require.NoError(t, tracker.runOnce(ctx)) var marketDeal model.Deal diff --git a/service/epochutil/epoch_test.go b/service/epochutil/epoch_test.go index dedf7b683..0d2298f03 100644 --- a/service/epochutil/epoch_test.go +++ b/service/epochutil/epoch_test.go @@ -9,20 +9,20 @@ import ( ) func TestDefaultValue(t *testing.T) { - err := Initialize(context.Background(), testutil.TestLotusAPI, "") + err := Initialize(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken) require.NoError(t, err) require.EqualValues(t, 1598306400, GenesisTimestamp) } func TestCalibNet(t *testing.T) { // This test may fail when calibnet resets - err := Initialize(context.Background(), testutil.CalibnetRPC, "") + err := Initialize(context.Background(), testutil.CalibnetRPC, testutil.TestLotusToken) require.NoError(t, err) require.EqualValues(t, 1667326380, GenesisTimestamp) } func TestEpochToTime(t *testing.T) { - err := Initialize(context.Background(), testutil.TestLotusAPI, "") + err := Initialize(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken) require.NoError(t, err) require.EqualValues(t, 1598306400, GenesisTimestamp) require.EqualValues(t, 1598306400, EpochToTime(0).Unix()) @@ -30,7 +30,7 @@ func TestEpochToTime(t *testing.T) { } func TestUnixToEpoch(t *testing.T) { - err := Initialize(context.Background(), testutil.TestLotusAPI, "") + err := Initialize(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken) require.NoError(t, err) require.EqualValues(t, 1598306400, GenesisTimestamp) require.EqualValues(t, 0, UnixToEpoch(1598306400)) @@ -38,7 +38,7 @@ func TestUnixToEpoch(t *testing.T) { } func TestTimeToEpoch(t *testing.T) { - err := Initialize(context.Background(), testutil.TestLotusAPI, "") + err := Initialize(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken) require.NoError(t, err) require.EqualValues(t, 1598306400, GenesisTimestamp) require.EqualValues(t, 0, TimeToEpoch(EpochToTime(0))) diff --git a/util/testutil/anvil.go b/util/testutil/anvil.go index bf5c10cbf..961923a05 100644 --- a/util/testutil/anvil.go +++ b/util/testutil/anvil.go @@ -41,11 +41,18 @@ func StartAnvil(t *testing.T, upstreamRPC string) *AnvilInstance { t.Fatalf("finding free port: %v", err) } - cmd := exec.Command("anvil", + args := []string{ "--fork-url", upstreamRPC, "--port", fmt.Sprintf("%d", port), "--silent", - ) + } + // forest gateway needs bearer auth; anvil supports --fork-header + haveTok := os.Getenv("FOREST_JWT") != "" + if haveTok { + args = append(args, "--fork-header", "Authorization: Bearer "+os.Getenv("FOREST_JWT")) + } + t.Logf("anvil forking %s (auth=%t)", upstreamRPC, haveTok) + cmd := exec.Command("anvil", args...) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr if err := cmd.Start(); err != nil { diff --git a/util/testutil/calibnet.go b/util/testutil/calibnet.go index a389a8bf1..b15b76edb 100644 --- a/util/testutil/calibnet.go +++ b/util/testutil/calibnet.go @@ -3,7 +3,7 @@ package testutil // calibnet constants shared by integration tests // CalibnetRPC defaults to the self-hosted forest gateway; glif rate-limited CI. -var CalibnetRPC = forestRPC("SINGULARITY_TEST_CALIBNET_RPC", "https://static.187.115.235.167.clients.your-server.de/calibnet/rpc/v1") +var CalibnetRPC = envOr("SINGULARITY_TEST_CALIBNET_RPC", "https://static.187.115.235.167.clients.your-server.de/calibnet/rpc/v1") const ( CalibnetChainID = 314159 diff --git a/util/testutil/testutils.go b/util/testutil/testutils.go index ee0fd0fb6..efa82d595 100644 --- a/util/testutil/testutils.go +++ b/util/testutil/testutils.go @@ -56,27 +56,36 @@ const TestWalletAddr = "f1fib3pv7jua2ockdugtz7viz3cyy6lkhh7rfx3sa" const TestPrivateKeyHex = "7b2254797065223a22736563703235366b31222c22507269766174654b6579223a226b35507976337148327349586343595a58594f5775453149326e32554539436861556b6c4e36695a5763453d227d" // TestLotusAPI defaults to the self-hosted forest gateway; glif rate-limited CI. -var TestLotusAPI = forestRPC("SINGULARITY_TEST_LOTUS_API", "https://static.187.115.235.167.clients.your-server.de/rpc/v1") +var TestLotusAPI = envOr("SINGULARITY_TEST_LOTUS_API", "https://static.187.115.235.167.clients.your-server.de/rpc/v1") -func forestRPC(override, base string) string { - if v := os.Getenv(override); v != "" { +// TestLotusToken is the bearer token for the forest gateway; empty in local +// dev and on fork PRs (see SkipIfNotExternalAPI). +var TestLotusToken = os.Getenv("FOREST_JWT") + +func envOr(name, def string) string { + if v := os.Getenv(name); v != "" { return v } - // gate rejects untokened requests; carry FOREST_JWT as a query param - if tok := os.Getenv("FOREST_JWT"); tok != "" { - return base + "?access_token=" + tok - } - return base + return def } -// SkipIfNotExternalAPI skips the test if SINGULARITY_TEST_EXTERNAL_API is not set -// Use this for tests that make external API calls (e.g., Lotus/Filecoin APIs) -// These are skipped by default because external APIs may be unreliable or rate-limited +// SkipIfNotExternalAPI skips the test if external API access isn't configured. +// Two conditions must hold: SINGULARITY_TEST_EXTERNAL_API opts in, and either +// FOREST_JWT is set or one of the endpoint overrides +// (SINGULARITY_TEST_LOTUS_API / SINGULARITY_TEST_CALIBNET_RPC) points at +// something that doesn't need the gateway's bearer auth. Fork PRs and local +// devs without a token skip cleanly instead of hitting a 401. func SkipIfNotExternalAPI(t *testing.T) { t.Helper() if os.Getenv("SINGULARITY_TEST_EXTERNAL_API") == "" { t.Skip("Skipping external API test. Set SINGULARITY_TEST_EXTERNAL_API=true to run") } + if os.Getenv("FOREST_JWT") == "" && + os.Getenv("SINGULARITY_TEST_LOTUS_API") == "" && + os.Getenv("SINGULARITY_TEST_CALIBNET_RPC") == "" { + t.Skip("Skipping: FOREST_JWT not set and no endpoint override") + } + t.Logf("external API: lotus=%s calibnet=%s token=%t", TestLotusAPI, CalibnetRPC, TestLotusToken != "") } func EscapePath(p string) string { diff --git a/util/util_test.go b/util/util_test.go index 80664e871..b182926d4 100644 --- a/util/util_test.go +++ b/util/util_test.go @@ -25,24 +25,15 @@ func TestNextPowerOfTwo(t *testing.T) { func TestNewLotusClient(t *testing.T) { testutil.SkipIfNotExternalAPI(t) - for _, token := range []string{""} { - t.Run(token, func(t *testing.T) { - client := NewLotusClient(testutil.TestLotusAPI, token) - resp, err := client.Call(context.Background(), "Filecoin.Version") - if token != "" { - require.Error(t, err) - require.ErrorContains(t, err, "401") - return - } - require.NoError(t, err) - require.NotNil(t, resp.Result) - }) - } + client := NewLotusClient(testutil.TestLotusAPI, testutil.TestLotusToken) + resp, err := client.Call(context.Background(), "Filecoin.Version") + require.NoError(t, err) + require.NotNil(t, resp.Result) } func TestGetLotusHeadTime(t *testing.T) { testutil.SkipIfNotExternalAPI(t) - headTime, err := GetLotusHeadTime(context.Background(), testutil.TestLotusAPI, "") + headTime, err := GetLotusHeadTime(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken) require.NoError(t, err) require.NotZero(t, headTime) } From 9a4bfd083bd4172fd7eea61064b06e27eed4f7b7 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Mon, 13 Jul 2026 14:27:13 +0200 Subject: [PATCH 3/3] initialize lastEpoch correctly --- service/dealtracker/dealtracker.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/service/dealtracker/dealtracker.go b/service/dealtracker/dealtracker.go index 2e6a1ea5a..15c5dcdc1 100644 --- a/service/dealtracker/dealtracker.go +++ b/service/dealtracker/dealtracker.go @@ -442,7 +442,9 @@ func (d *DealTracker) runOnce(ctx context.Context) error { return errors.Wrapf(err, "failed to get lotus head time from %s", d.lotusURL) } - var lastEpoch int32 + // derive from head so expiration works even if the market stream is empty; + // the trackDeal callback can still bump it higher if a deal reports newer + lastEpoch := int32(epochutil.TimeToEpoch(headTime)) db := d.dbNoContext.WithContext(ctx) var actors []model.Actor