diff --git a/.github/workflows/backups-dashboards.yml b/.github/workflows/backups-dashboards.yml new file mode 100644 index 00000000000..a30f72c94e8 --- /dev/null +++ b/.github/workflows/backups-dashboards.yml @@ -0,0 +1,112 @@ +# Author: Michele@DevOpsTeam.Erigon +# Maintainers: DevOps@DevOpsTeam.Erigon +# Description: This workflow is responsible for backing up the dashboards of the Erigon project. +# Status: Mergeable for testing + +name: Dashboards backup + +env: + GREEN: '\033[0;32m' + RED: '\033[0;31m' + YELLOW: '\033[0;33m' + NOCOLOUR: '\033[0m' + ERIGON_BRANCH: main + +on: + workflow_dispatch: + inputs: + TEMPLATE_BRANCH: + required: false + type: string + default: main + description: 'The branch to pull the backup script from (default: main)' + +jobs: + # + # NOTE: This workflow splits the backup process in 3 jobs even though 2 (or even 1 if you're particularly naive) would have been enough. I decided to do that just as a reference for passing data between jobs. The overhead is minimal. + # + preparation: + runs-on: ubuntu-latest + environment: dashboards_backups + steps: + - name: Pull backup script from ${{ inputs.TEMPLATE_BRANCH}} branch + run: | + set +x + curl -L -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" -H "Accept: application/vnd.github.v3.raw" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/erigontech/scripts/contents/dashboards/dashboard-backup.sh?ref=${{ inputs.TEMPLATE_BRANCH}} -o /tmp/dashboard-backup + + - name: Upload dashboard-backup + uses: actions/upload-artifact@v3 + with: + name: dashboard-backup + path: /tmp/dashboard-backup + retention-days: 1 + + + backup_dashboard: + needs: preparation + runs-on: ubuntu-latest + environment: dashboards_backups + strategy: + matrix: + # For each dashboard add an entry in the list here below (and add an entry in the dictionary within the backup script too!) + dashboard: [erigon_custom_metrics] + env: + DASHBOARDS_AUTH_TOKEN: ${{ secrets.DASHBOARDS_AUTH_TOKEN }} + GH_TOKEN: ${{ secrets.GH_TOKEN }} + DASHBOARDS_GIT_CONFIG: ${{ secrets.DASHBOARDS_GIT_CONFIG }} + steps: + - name: Clone erigon from ${{ env.ERIGON_BRANCH }} branch + uses: actions/checkout@v4 + with: + ref: ${{ env.ERIGON_BRANCH }} + fetch-depth: 1 + path: ${{ github.workspace }}/erigon + + - name: Move repository to $HOME # as the backup script expects it to be there + run: mv ${{ github.workspace }}/erigon $HOME/ + + - name: Set-up git + run: | + echo ${{ secrets.DASHBOARDS_GIT_CONFIG }} | base64 -d > $HOME/.gitconfig + + - name: Download dashboard-backup + uses: actions/download-artifact@v3 + with: + name: dashboard-backup + path: /tmp + + - name: Set dashboard-backup permissions + run: chmod +x /tmp/dashboard-backup + + - name: BACKUP OF ${{ matrix.dashboard }} DASHBOARD + run: | + set +x + echo -e "${{ env.GREEN }} I'm processing the ${{ matrix.dashboard }} dashboard... ${{ env.NOCOLOUR }}" + /tmp/dashboard-backup ${{ matrix.dashboard }} + + + pulizie_di_primavera: + needs: [preparation, backup_dashboard] + runs-on: ubuntu-latest + if: always() + steps: + - name: housekeeping + uses: geekyeggo/delete-artifact@v2 + with: + name: | + dashboard-backup + + - name: Bye ${{ github.actor }} + if: always() + run: | + set +x + if [ "${{ needs.preparation.result }}" == "success" ] && [ "${{ needs.backup_dashboard.result }}" == "success" ]; then + echo -e "${{ env.YELLOW }} --------------------------------------------------------------------------------------------------------------------- ${{ env.NOCOLOUR }}" + echo -e "${{ env.YELLOW }} *** It was a true pleasure to serve you ${{ github.actor }}, SEE YOU NEXT TIME! ${{ env.NOCOLOUR }}" + echo -e "${{ env.YELLOW }} --------------------------------------------------------------------------------------------------------------------- ${{ env.NOCOLOUR }}" + else + echo -e "${{ env.RED }} --------------------------------------------------------------------------------------------------------------------- ${{ env.RED }}" + echo -e "${{ env.RED }} *** Sorry ${{ github.actor }}, it seems there was an error this time: just fix it and COME BACK! ${{ env.RED }}" + echo -e "${{ env.RED }} --------------------------------------------------------------------------------------------------------------------- ${{ env.RED }}" + fi + \ No newline at end of file diff --git a/.github/workflows/manifest.yml b/.github/workflows/manifest.yml index ee14b9f153e..13c054ed578 100644 --- a/.github/workflows/manifest.yml +++ b/.github/workflows/manifest.yml @@ -31,9 +31,9 @@ jobs: go-version: '1.23' - run: make downloader - run: echo $ModModified - - run: ./build/bin/downloader manifest-verify --chain mainnet - - run: ./build/bin/downloader manifest-verify --chain bor-mainnet - - run: ./build/bin/downloader manifest-verify --chain gnosis - - run: ./build/bin/downloader manifest-verify --chain chiado - - run: ./build/bin/downloader manifest-verify --chain sepolia - - run: ./build/bin/downloader manifest-verify --chain amoy + - run: ./build/bin/downloader manifest-verify --chain mainnet --webseed 'https://erigon3-v1-snapshots-mainnet.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain bor-mainnet --webseed 'https://erigon3-v1-snapshots-bor-mainnet.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain gnosis --webseed 'https://erigon3-v1-snapshots-gnosis.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain chiado --webseed 'https://erigon3-v1-snapshots-chiado.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain sepolia --webseed 'https://erigon3-v1-snapshots-sepolia.erigon.network' + - run: ./build/bin/downloader manifest-verify --chain amoy --webseed 'https://erigon3-v1-snapshots-amoy.erigon.network' diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 63c7e8fdb3b..b1e1de4bed0 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -154,7 +154,7 @@ require ( go.opentelemetry.io/otel v1.8.0 // indirect go.opentelemetry.io/otel/trace v1.8.0 // indirect go.uber.org/goleak v1.3.0 // indirect - golang.org/x/net v0.30.0 + golang.org/x/net v0.33.0 golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 26ad7c8e9c1..c2f8d9dda5d 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -556,8 +556,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/go.mod b/go.mod index cc2d29e043f..863bd377acb 100644 --- a/go.mod +++ b/go.mod @@ -92,7 +92,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/crypto v0.31.0 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 golang.org/x/sync v0.10.0 golang.org/x/sys v0.28.0 golang.org/x/time v0.8.0 diff --git a/go.sum b/go.sum index c7883eb4b20..9767051df27 100644 --- a/go.sum +++ b/go.sum @@ -1060,8 +1060,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/turbo/snapshotsync/snapshots.go b/turbo/snapshotsync/snapshots.go index c46d4a5ff02..6ecdc947dcf 100644 --- a/turbo/snapshotsync/snapshots.go +++ b/turbo/snapshotsync/snapshots.go @@ -35,7 +35,6 @@ import ( "github.com/erigontech/erigon-lib/common/background" "github.com/erigontech/erigon-lib/common/datadir" "github.com/erigontech/erigon-lib/common/dbg" - dir2 "github.com/erigontech/erigon-lib/common/dir" "github.com/erigontech/erigon-lib/diagnostics" "github.com/erigontech/erigon-lib/downloader/snaptype" "github.com/erigontech/erigon-lib/log/v3" @@ -56,6 +55,10 @@ type SortedRange interface { // NoOverlaps - keep largest ranges and avoid overlap func NoOverlaps[T SortedRange](in []T) (res []T) { + if len(in) == 1 { + return in + } + for i := 0; i < len(in); i++ { r := in[i] iFrom, iTo := r.GetRange() @@ -83,6 +86,10 @@ func NoGaps[T SortedRange](in []T) (out []T, missingRanges []Range) { if len(in) == 0 { return nil, nil } + if len(in) == 1 { + return in, nil + } + prevTo, _ := in[0].GetRange() for _, f := range in { from, to := f.GetRange() @@ -980,10 +987,6 @@ func (s *RoSnapshots) InitSegments(fileNames []string) error { } func TypedSegments(dir string, minBlock uint64, types []snaptype.Type, allowGaps bool) (res []snaptype.FileInfo, missingSnapshots []Range, err error) { - segmentsTypeCheck := func(dir string, in []snaptype.FileInfo) (res []snaptype.FileInfo) { - return typeOfSegmentsMustExist(dir, in, types) - } - list, err := snaptype.Segments(dir) if err != nil { @@ -1002,10 +1005,11 @@ func TypedSegments(dir string, minBlock uint64, types []snaptype.Type, allowGaps } if allowGaps { - l = NoOverlaps(segmentsTypeCheck(dir, l)) + l = NoOverlaps(l) } else { - l, m = NoGaps(NoOverlaps(segmentsTypeCheck(dir, l))) + l, m = NoGaps(NoOverlaps(l)) } + if len(m) > 0 { lst := m[len(m)-1] log.Debug("[snapshots] see gap", "type", segType, "from", lst.from) @@ -1619,28 +1623,6 @@ func sendDiagnostics(startIndexingTime time.Time, indexPercent map[string]int, a }) } -func typeOfSegmentsMustExist(dir string, in []snaptype.FileInfo, types []snaptype.Type) (res []snaptype.FileInfo) { -MainLoop: - for _, f := range in { - if f.From == f.To { - continue - } - for _, t := range types { - p := filepath.Join(dir, snaptype.SegmentFileName(f.Version, f.From, f.To, t.Enum())) - exists, err := dir2.FileExist(p) - if err != nil { - log.Debug("[snapshots] FileExist error", "err", err, "path", p) - continue MainLoop - } - if !exists { - continue MainLoop - } - res = append(res, f) - } - } - return res -} - func removeOldFiles(toDel []string, snapDir string) { for _, f := range toDel { _ = os.Remove(f)