Skip to content

Commit

Permalink
Merge branch 'main' into r_2
Browse files Browse the repository at this point in the history
  • Loading branch information
AskAlexSharov committed Dec 20, 2024
2 parents a24ba43 + ee9d3fc commit 0cf39bb
Show file tree
Hide file tree
Showing 7 changed files with 135 additions and 41 deletions.
112 changes: 112 additions & 0 deletions .github/workflows/backups-dashboards.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
# Author: [email protected]
# Maintainers: [email protected]
# Description: This workflow is responsible for backing up the dashboards of the Erigon project.
# Status: Mergeable for testing

name: Dashboards backup

env:
GREEN: '\033[0;32m'
RED: '\033[0;31m'
YELLOW: '\033[0;33m'
NOCOLOUR: '\033[0m'
ERIGON_BRANCH: main

on:
workflow_dispatch:
inputs:
TEMPLATE_BRANCH:
required: false
type: string
default: main
description: 'The branch to pull the backup script from (default: main)'

jobs:
#
# NOTE: This workflow splits the backup process in 3 jobs even though 2 (or even 1 if you're particularly naive) would have been enough. I decided to do that just as a reference for passing data between jobs. The overhead is minimal.
#
preparation:
runs-on: ubuntu-latest
environment: dashboards_backups
steps:
- name: Pull backup script from ${{ inputs.TEMPLATE_BRANCH}} branch
run: |
set +x
curl -L -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" -H "Accept: application/vnd.github.v3.raw" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/erigontech/scripts/contents/dashboards/dashboard-backup.sh?ref=${{ inputs.TEMPLATE_BRANCH}} -o /tmp/dashboard-backup
- name: Upload dashboard-backup
uses: actions/upload-artifact@v3
with:
name: dashboard-backup
path: /tmp/dashboard-backup
retention-days: 1


backup_dashboard:
needs: preparation
runs-on: ubuntu-latest
environment: dashboards_backups
strategy:
matrix:
# For each dashboard add an entry in the list here below (and add an entry in the dictionary within the backup script too!)
dashboard: [erigon_custom_metrics]
env:
DASHBOARDS_AUTH_TOKEN: ${{ secrets.DASHBOARDS_AUTH_TOKEN }}
GH_TOKEN: ${{ secrets.GH_TOKEN }}
DASHBOARDS_GIT_CONFIG: ${{ secrets.DASHBOARDS_GIT_CONFIG }}
steps:
- name: Clone erigon from ${{ env.ERIGON_BRANCH }} branch
uses: actions/checkout@v4
with:
ref: ${{ env.ERIGON_BRANCH }}
fetch-depth: 1
path: ${{ github.workspace }}/erigon

- name: Move repository to $HOME # as the backup script expects it to be there
run: mv ${{ github.workspace }}/erigon $HOME/

- name: Set-up git
run: |
echo ${{ secrets.DASHBOARDS_GIT_CONFIG }} | base64 -d > $HOME/.gitconfig
- name: Download dashboard-backup
uses: actions/download-artifact@v3
with:
name: dashboard-backup
path: /tmp

- name: Set dashboard-backup permissions
run: chmod +x /tmp/dashboard-backup

- name: BACKUP OF ${{ matrix.dashboard }} DASHBOARD
run: |
set +x
echo -e "${{ env.GREEN }} I'm processing the ${{ matrix.dashboard }} dashboard... ${{ env.NOCOLOUR }}"
/tmp/dashboard-backup ${{ matrix.dashboard }}

pulizie_di_primavera:
needs: [preparation, backup_dashboard]
runs-on: ubuntu-latest
if: always()
steps:
- name: housekeeping
uses: geekyeggo/delete-artifact@v2
with:
name: |
dashboard-backup
- name: Bye ${{ github.actor }}
if: always()
run: |
set +x
if [ "${{ needs.preparation.result }}" == "success" ] && [ "${{ needs.backup_dashboard.result }}" == "success" ]; then
echo -e "${{ env.YELLOW }} --------------------------------------------------------------------------------------------------------------------- ${{ env.NOCOLOUR }}"
echo -e "${{ env.YELLOW }} *** It was a true pleasure to serve you ${{ github.actor }}, SEE YOU NEXT TIME! ${{ env.NOCOLOUR }}"
echo -e "${{ env.YELLOW }} --------------------------------------------------------------------------------------------------------------------- ${{ env.NOCOLOUR }}"
else
echo -e "${{ env.RED }} --------------------------------------------------------------------------------------------------------------------- ${{ env.RED }}"
echo -e "${{ env.RED }} *** Sorry ${{ github.actor }}, it seems there was an error this time: just fix it and COME BACK! ${{ env.RED }}"
echo -e "${{ env.RED }} --------------------------------------------------------------------------------------------------------------------- ${{ env.RED }}"
fi
12 changes: 6 additions & 6 deletions .github/workflows/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ jobs:
go-version: '1.23'
- run: make downloader
- run: echo $ModModified
- run: ./build/bin/downloader manifest-verify --chain mainnet
- run: ./build/bin/downloader manifest-verify --chain bor-mainnet
- run: ./build/bin/downloader manifest-verify --chain gnosis
- run: ./build/bin/downloader manifest-verify --chain chiado
- run: ./build/bin/downloader manifest-verify --chain sepolia
- run: ./build/bin/downloader manifest-verify --chain amoy
- run: ./build/bin/downloader manifest-verify --chain mainnet --webseed 'https://erigon3-v1-snapshots-mainnet.erigon.network'
- run: ./build/bin/downloader manifest-verify --chain bor-mainnet --webseed 'https://erigon3-v1-snapshots-bor-mainnet.erigon.network'
- run: ./build/bin/downloader manifest-verify --chain gnosis --webseed 'https://erigon3-v1-snapshots-gnosis.erigon.network'
- run: ./build/bin/downloader manifest-verify --chain chiado --webseed 'https://erigon3-v1-snapshots-chiado.erigon.network'
- run: ./build/bin/downloader manifest-verify --chain sepolia --webseed 'https://erigon3-v1-snapshots-sepolia.erigon.network'
- run: ./build/bin/downloader manifest-verify --chain amoy --webseed 'https://erigon3-v1-snapshots-amoy.erigon.network'
2 changes: 1 addition & 1 deletion erigon-lib/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ require (
go.opentelemetry.io/otel v1.8.0 // indirect
go.opentelemetry.io/otel/trace v1.8.0 // indirect
go.uber.org/goleak v1.3.0 // indirect
golang.org/x/net v0.30.0
golang.org/x/net v0.33.0
golang.org/x/text v0.21.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/mathutil v1.6.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions erigon-lib/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -556,8 +556,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ require (
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.31.0
golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
golang.org/x/net v0.32.0
golang.org/x/net v0.33.0
golang.org/x/sync v0.10.0
golang.org/x/sys v0.28.0
golang.org/x/time v0.8.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1060,8 +1060,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down
40 changes: 11 additions & 29 deletions turbo/snapshotsync/snapshots.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ import (
"github.com/erigontech/erigon-lib/common/background"
"github.com/erigontech/erigon-lib/common/datadir"
"github.com/erigontech/erigon-lib/common/dbg"
dir2 "github.com/erigontech/erigon-lib/common/dir"
"github.com/erigontech/erigon-lib/diagnostics"
"github.com/erigontech/erigon-lib/downloader/snaptype"
"github.com/erigontech/erigon-lib/log/v3"
Expand All @@ -56,6 +55,10 @@ type SortedRange interface {

// NoOverlaps - keep largest ranges and avoid overlap
func NoOverlaps[T SortedRange](in []T) (res []T) {
if len(in) == 1 {
return in
}

for i := 0; i < len(in); i++ {
r := in[i]
iFrom, iTo := r.GetRange()
Expand Down Expand Up @@ -83,6 +86,10 @@ func NoGaps[T SortedRange](in []T) (out []T, missingRanges []Range) {
if len(in) == 0 {
return nil, nil
}
if len(in) == 1 {
return in, nil
}

prevTo, _ := in[0].GetRange()
for _, f := range in {
from, to := f.GetRange()
Expand Down Expand Up @@ -980,10 +987,6 @@ func (s *RoSnapshots) InitSegments(fileNames []string) error {
}

func TypedSegments(dir string, minBlock uint64, types []snaptype.Type, allowGaps bool) (res []snaptype.FileInfo, missingSnapshots []Range, err error) {
segmentsTypeCheck := func(dir string, in []snaptype.FileInfo) (res []snaptype.FileInfo) {
return typeOfSegmentsMustExist(dir, in, types)
}

list, err := snaptype.Segments(dir)

if err != nil {
Expand All @@ -1002,10 +1005,11 @@ func TypedSegments(dir string, minBlock uint64, types []snaptype.Type, allowGaps
}

if allowGaps {
l = NoOverlaps(segmentsTypeCheck(dir, l))
l = NoOverlaps(l)
} else {
l, m = NoGaps(NoOverlaps(segmentsTypeCheck(dir, l)))
l, m = NoGaps(NoOverlaps(l))
}

if len(m) > 0 {
lst := m[len(m)-1]
log.Debug("[snapshots] see gap", "type", segType, "from", lst.from)
Expand Down Expand Up @@ -1619,28 +1623,6 @@ func sendDiagnostics(startIndexingTime time.Time, indexPercent map[string]int, a
})
}

func typeOfSegmentsMustExist(dir string, in []snaptype.FileInfo, types []snaptype.Type) (res []snaptype.FileInfo) {
MainLoop:
for _, f := range in {
if f.From == f.To {
continue
}
for _, t := range types {
p := filepath.Join(dir, snaptype.SegmentFileName(f.Version, f.From, f.To, t.Enum()))
exists, err := dir2.FileExist(p)
if err != nil {
log.Debug("[snapshots] FileExist error", "err", err, "path", p)
continue MainLoop
}
if !exists {
continue MainLoop
}
res = append(res, f)
}
}
return res
}

func removeOldFiles(toDel []string, snapDir string) {
for _, f := range toDel {
_ = os.Remove(f)
Expand Down

0 comments on commit 0cf39bb

Please sign in to comment.