From b408f78d9472ebc4e01fdf98560ff6dc4389d0f1 Mon Sep 17 00:00:00 2001 From: Ostroukhov Nikita Date: Thu, 19 Dec 2024 04:28:56 +0000 Subject: [PATCH 1/5] Fixed snapshot read from disk (#13144) --- turbo/snapshotsync/snapshots.go | 40 +++++++++------------------------ 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/turbo/snapshotsync/snapshots.go b/turbo/snapshotsync/snapshots.go index c46d4a5ff02..6ecdc947dcf 100644 --- a/turbo/snapshotsync/snapshots.go +++ b/turbo/snapshotsync/snapshots.go @@ -35,7 +35,6 @@ import ( "github.com/erigontech/erigon-lib/common/background" "github.com/erigontech/erigon-lib/common/datadir" "github.com/erigontech/erigon-lib/common/dbg" - dir2 "github.com/erigontech/erigon-lib/common/dir" "github.com/erigontech/erigon-lib/diagnostics" "github.com/erigontech/erigon-lib/downloader/snaptype" "github.com/erigontech/erigon-lib/log/v3" @@ -56,6 +55,10 @@ type SortedRange interface { // NoOverlaps - keep largest ranges and avoid overlap func NoOverlaps[T SortedRange](in []T) (res []T) { + if len(in) == 1 { + return in + } + for i := 0; i < len(in); i++ { r := in[i] iFrom, iTo := r.GetRange() @@ -83,6 +86,10 @@ func NoGaps[T SortedRange](in []T) (out []T, missingRanges []Range) { if len(in) == 0 { return nil, nil } + if len(in) == 1 { + return in, nil + } + prevTo, _ := in[0].GetRange() for _, f := range in { from, to := f.GetRange() @@ -980,10 +987,6 @@ func (s *RoSnapshots) InitSegments(fileNames []string) error { } func TypedSegments(dir string, minBlock uint64, types []snaptype.Type, allowGaps bool) (res []snaptype.FileInfo, missingSnapshots []Range, err error) { - segmentsTypeCheck := func(dir string, in []snaptype.FileInfo) (res []snaptype.FileInfo) { - return typeOfSegmentsMustExist(dir, in, types) - } - list, err := snaptype.Segments(dir) if err != nil { @@ -1002,10 +1005,11 @@ func TypedSegments(dir string, minBlock uint64, types []snaptype.Type, allowGaps } if allowGaps { - l = NoOverlaps(segmentsTypeCheck(dir, l)) + l = NoOverlaps(l) } else { - l, m = NoGaps(NoOverlaps(segmentsTypeCheck(dir, l))) + l, m = NoGaps(NoOverlaps(l)) } + if len(m) > 0 { lst := m[len(m)-1] log.Debug("[snapshots] see gap", "type", segType, "from", lst.from) @@ -1619,28 +1623,6 @@ func sendDiagnostics(startIndexingTime time.Time, indexPercent map[string]int, a }) } -func typeOfSegmentsMustExist(dir string, in []snaptype.FileInfo, types []snaptype.Type) (res []snaptype.FileInfo) { -MainLoop: - for _, f := range in { - if f.From == f.To { - continue - } - for _, t := range types { - p := filepath.Join(dir, snaptype.SegmentFileName(f.Version, f.From, f.To, t.Enum())) - exists, err := dir2.FileExist(p) - if err != nil { - log.Debug("[snapshots] FileExist error", "err", err, "path", p) - continue MainLoop - } - if !exists { - continue MainLoop - } - res = append(res, f) - } - } - return res -} - func removeOldFiles(toDel []string, snapDir string) { for _, f := range toDel { _ = os.Remove(f) From 94a1581a5d5fce01fa1fedc1f4e8a33c2c4f1773 Mon Sep 17 00:00:00 2001 From: Andrew Ashikhmin <34320705+yperbasis@users.noreply.github.com> Date: Thu, 19 Dec 2024 11:14:57 +0100 Subject: [PATCH 2/5] deps: upgrade golang.org/x/net to v0.33.0 (#13177) --- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index d337d90d9f7..b46410c6a3a 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -153,7 +153,7 @@ require ( go.opentelemetry.io/otel v1.8.0 // indirect go.opentelemetry.io/otel/trace v1.8.0 // indirect go.uber.org/goleak v1.3.0 // indirect - golang.org/x/net v0.30.0 + golang.org/x/net v0.33.0 golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 150d86f95c5..65818353ba7 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -554,8 +554,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/go.mod b/go.mod index 07949c43314..b1d51ea37a2 100644 --- a/go.mod +++ b/go.mod @@ -92,7 +92,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/crypto v0.31.0 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 golang.org/x/sync v0.10.0 golang.org/x/sys v0.28.0 golang.org/x/time v0.8.0 diff --git a/go.sum b/go.sum index e58d9b5bbe3..ff4b776ceda 100644 --- a/go.sum +++ b/go.sum @@ -1058,8 +1058,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= From a93dd01b7550a0289b9d5db4c168ca88ceaf7f93 Mon Sep 17 00:00:00 2001 From: Michele Modolo <70838029+michelemodolo@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:30:44 +0100 Subject: [PATCH 3/5] Selective check for manifest (#13179) This restricts the manifest check to those sources which are actually contributing to snapshots distribution. --- .github/workflows/manifest.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/manifest.yml b/.github/workflows/manifest.yml index ee14b9f153e..13c054ed578 100644 --- a/.github/workflows/manifest.yml +++ b/.github/workflows/manifest.yml @@ -31,9 +31,9 @@ jobs: go-version: '1.23' - run: make downloader - run: echo $ModModified - - run: ./build/bin/downloader manifest-verify --chain mainnet - - run: ./build/bin/downloader manifest-verify --chain bor-mainnet - - run: ./build/bin/downloader manifest-verify --chain gnosis - - run: ./build/bin/downloader manifest-verify --chain chiado - - run: ./build/bin/downloader manifest-verify --chain sepolia - - run: ./build/bin/downloader manifest-verify --chain amoy + - run: ./build/bin/downloader manifest-verify --chain mainnet --webseed 'https://erigon3-v1-snapshots-mainnet.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain bor-mainnet --webseed 'https://erigon3-v1-snapshots-bor-mainnet.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain gnosis --webseed 'https://erigon3-v1-snapshots-gnosis.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain chiado --webseed 'https://erigon3-v1-snapshots-chiado.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain sepolia --webseed 'https://erigon3-v1-snapshots-sepolia.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain amoy --webseed 'https://erigon3-v1-snapshots-amoy.erigon.network' From c6bf543d5db5c65bf23b091b29c1e1e3c19b59e4 Mon Sep 17 00:00:00 2001 From: Michele Modolo <70838029+michelemodolo@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:47:25 +0100 Subject: [PATCH 4/5] Automation for dashboards backup (#13182) This introduces a new workflow for automating the dashboards backup. --------- Co-authored-by: Michele Modolo --- .github/workflows/backups-dashboards.yml | 130 +++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 .github/workflows/backups-dashboards.yml diff --git a/.github/workflows/backups-dashboards.yml b/.github/workflows/backups-dashboards.yml new file mode 100644 index 00000000000..1533b3d2664 --- /dev/null +++ b/.github/workflows/backups-dashboards.yml @@ -0,0 +1,130 @@ +# Author: Michele@DevOpsTeam.Erigon +# Maintainers: DevOps@DevOpsTeam.Erigon +# Description: This workflow is responsible for backing up the dashboards of the Erigon project. +# Status: Mergeable for testing + +name: Dashboards backup + +env: + GREEN: '\033[0;32m' + RED: '\033[0;31m' + YELLOW: '\033[0;33m' + NOCOLOUR: '\033[0m' + +on: + workflow_dispatch: + inputs: + TEMPLATE_BRANCH: + required: false + type: string + default: main + description: 'The branch to pull the backup script from (default: main)' + ERIGON_BRANCH: + required: false + type: string + default: main + description: 'The branch to pull the erigon repo from (default: main)' + +jobs: + # + # NOTE: This workflow splits the backup process in 3 jobs even though 2 (or even 1 if you're particularly naive) would have been enough. I decided to do that just as a reference for passing data between jobs. The overhead is minimal. + # + preparation: + runs-on: ubuntu-latest + environment: dashboards_backups + steps: + - name: CLONE THE ERIGON REPO from ${{ inputs.ERIGON_BRANCH }} branch + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ERIGON_BRANCH }} + fetch-depth: 1 + path: ${{ github.workspace }}/erigon + + - name: Upload Erigon repository + uses: actions/upload-artifact@v3 + with: + name: erigon-repo + path: ${{ github.workspace }}/erigon + retention-days: 1 + + - name: PULL THE LATEST VERSION OF BACKUP SCRIPT from ${{ inputs.TEMPLATE_BRANCH}} branch + run: | + set +x + curl -L -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" -H "Accept: application/vnd.github.v3.raw" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/erigontech/scripts/contents/dashboards/dashboard-backup.sh?ref=${{ inputs.TEMPLATE_BRANCH}} -o /tmp/dashboard-backup + + - name: Upload dashboard-backup + uses: actions/upload-artifact@v3 + with: + name: dashboard-backup + path: /tmp/dashboard-backup + retention-days: 1 + + + backup_dashboard: + needs: preparation + runs-on: ubuntu-latest + environment: dashboards_backups + strategy: + matrix: + # For each dashboard add an entry in the list here below (and add an entry in the dictionary within the backup script too!) + dashboard: [erigon_custom_metrics] + env: + DASHBOARDS_AUTH_TOKEN: ${{ secrets.DASHBOARDS_AUTH_TOKEN }} + GH_TOKEN: ${{ secrets.GH_TOKEN }} + DASHBOARDS_GIT_CONFIG: ${{ secrets.DASHBOARDS_GIT_CONFIG }} + steps: + - name: SET-UP GIT CONFIG + run: | + echo ${{ secrets.DASHBOARDS_GIT_CONFIG }} | base64 -d > $HOME/.gitconfig + + - name: Download Erigon repository + uses: actions/download-artifact@v3 + with: + name: erigon-repo + path: ${{ github.workspace }}/erigon + + - name: Move Erigon repository to $HOME # as the backup script expects it to be there + run: mv ${{ github.workspace }}/erigon $HOME/ + + - name: Download dashboard-backup + uses: actions/download-artifact@v3 + with: + name: dashboard-backup + path: /tmp + + - name: Set dashboard-backup permissions + run: chmod +x /tmp/dashboard-backup + + - name: BACKUP OF ${{ matrix.dashboard }} DASHBOARD + run: | + set +x + echo -e "${{ env.GREEN }} I'm processing the ${{ matrix.dashboard }} dashboard... ${{ env.NOCOLOUR }}" + /tmp/dashboard-backup ${{ matrix.dashboard }} + + + pulizie_di_primavera: + needs: [preparation, backup_dashboard] + runs-on: ubuntu-latest + if: always() + steps: + - name: housekeeping + uses: geekyeggo/delete-artifact@v2 + with: + name: | + dashboard-backup + erigon-repo + + - name: Bye ${{ github.actor }} + if: always() + run: | + set +x + if [ "${{ needs.preparation.result }}" == "success" ] && [ "${{ needs.backup_dashboard.result }}" == "success" ]; then + echo -e "${{ env.YELLOW }} --------------------------------------------------------------------------------------------------------------------- ${{ env.NOCOLOUR }}" + echo -e "${{ env.YELLOW }} *** It was a true pleasure to serve you ${{ github.actor }}, SEE YOU NEXT TIME! ${{ env.NOCOLOUR }}" + echo -e "${{ env.YELLOW }} --------------------------------------------------------------------------------------------------------------------- ${{ env.NOCOLOUR }}" + else + echo -e "${{ env.RED }} --------------------------------------------------------------------------------------------------------------------- ${{ env.RED }}" + echo -e "${{ env.RED }} *** Sorry ${{ github.actor }}, it seems there was an error this time: just fix it and COME BACK! ${{ env.RED }}" + echo -e "${{ env.RED }} --------------------------------------------------------------------------------------------------------------------- ${{ env.RED }}" + fi + \ No newline at end of file From ee9d3fccb6083af73209e5a1f5afdac70a82cb5d Mon Sep 17 00:00:00 2001 From: Michele Modolo <70838029+michelemodolo@users.noreply.github.com> Date: Thu, 19 Dec 2024 18:45:25 +0100 Subject: [PATCH 5/5] dashboards backup (#13186) I streamlined the workflow for backing up dashboards and avoided unnecessary fiddling of users with erigon repo branch. --- .github/workflows/backups-dashboards.yml | 46 ++++++++---------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/.github/workflows/backups-dashboards.yml b/.github/workflows/backups-dashboards.yml index 1533b3d2664..a30f72c94e8 100644 --- a/.github/workflows/backups-dashboards.yml +++ b/.github/workflows/backups-dashboards.yml @@ -10,6 +10,7 @@ env: RED: '\033[0;31m' YELLOW: '\033[0;33m' NOCOLOUR: '\033[0m' + ERIGON_BRANCH: main on: workflow_dispatch: @@ -19,11 +20,6 @@ on: type: string default: main description: 'The branch to pull the backup script from (default: main)' - ERIGON_BRANCH: - required: false - type: string - default: main - description: 'The branch to pull the erigon repo from (default: main)' jobs: # @@ -32,22 +28,8 @@ jobs: preparation: runs-on: ubuntu-latest environment: dashboards_backups - steps: - - name: CLONE THE ERIGON REPO from ${{ inputs.ERIGON_BRANCH }} branch - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ERIGON_BRANCH }} - fetch-depth: 1 - path: ${{ github.workspace }}/erigon - - - name: Upload Erigon repository - uses: actions/upload-artifact@v3 - with: - name: erigon-repo - path: ${{ github.workspace }}/erigon - retention-days: 1 - - - name: PULL THE LATEST VERSION OF BACKUP SCRIPT from ${{ inputs.TEMPLATE_BRANCH}} branch + steps: + - name: Pull backup script from ${{ inputs.TEMPLATE_BRANCH}} branch run: | set +x curl -L -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" -H "Accept: application/vnd.github.v3.raw" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/erigontech/scripts/contents/dashboards/dashboard-backup.sh?ref=${{ inputs.TEMPLATE_BRANCH}} -o /tmp/dashboard-backup @@ -72,19 +54,20 @@ jobs: DASHBOARDS_AUTH_TOKEN: ${{ secrets.DASHBOARDS_AUTH_TOKEN }} GH_TOKEN: ${{ secrets.GH_TOKEN }} DASHBOARDS_GIT_CONFIG: ${{ secrets.DASHBOARDS_GIT_CONFIG }} - steps: - - name: SET-UP GIT CONFIG - run: | - echo ${{ secrets.DASHBOARDS_GIT_CONFIG }} | base64 -d > $HOME/.gitconfig - - - name: Download Erigon repository - uses: actions/download-artifact@v3 + steps: + - name: Clone erigon from ${{ env.ERIGON_BRANCH }} branch + uses: actions/checkout@v4 with: - name: erigon-repo - path: ${{ github.workspace }}/erigon + ref: ${{ env.ERIGON_BRANCH }} + fetch-depth: 1 + path: ${{ github.workspace }}/erigon - - name: Move Erigon repository to $HOME # as the backup script expects it to be there + - name: Move repository to $HOME # as the backup script expects it to be there run: mv ${{ github.workspace }}/erigon $HOME/ + + - name: Set-up git + run: | + echo ${{ secrets.DASHBOARDS_GIT_CONFIG }} | base64 -d > $HOME/.gitconfig - name: Download dashboard-backup uses: actions/download-artifact@v3 @@ -112,7 +95,6 @@ jobs: with: name: | dashboard-backup - erigon-repo - name: Bye ${{ github.actor }} if: always()