no more lotus build (#63)

* forestTest

* docsgen

* handle stragglers

* added existing APIs already available in Forest

* no more lotus build

* four hodlers left

* renames and reduction

* merge fix

* no more lotus build

* four hodlers left

* some

* aleret wiring, fullnode alternative

* goamd64

* lotus changed, so change version sig, drop v1api, etc

* in-house alertinginterface

* complete rm lotus/build for curio

* sptool needs build, so test cannot pass

* merge fixes

* use specific subset of chain api

* Use specific subset of chain API

* unused

* taskAlert

* tidy

* cbor  thing

* tidy again

* gen needs tidy

* fixes

* tidy

* avoid gen-check additions

.circleci/config.yml

@@ -232,7 +232,6 @@ jobs:
       - attach_workspace:
           at: ~/
       - run: go install golang.org/x/tools/cmd/goimports
-      - run: go install github.com/hannahhoward/cbor-gen-for
       - run: make gen
       - run: git --no-pager diff && git --no-pager diff --quiet
 

.gitignore

@@ -2,6 +2,7 @@
 /sptool
 /docgen-md
 /docgen-openrpc
+/deps.json
 
 extern/filecoin-ffi/rust/target
 **/*.a
@@ -16,6 +17,9 @@ build/.*
 /venv
 scratchpad
 
+# to ensure tests build: itests/ go test -c .
+/itests/itests.test
+
 # The following files are checked into git and result
 # in dirty git state if removed from the docker context
 !extern/filecoin-ffi/rust/filecoin.pc

Makefile

@@ -37,9 +37,11 @@ CLEAN+=build/.update-modules
 deps: $(BUILD_DEPS)
 .PHONY: deps
 
+## ldflags -s -w strips binary
+
 curio: $(BUILD_DEPS)
 	rm -f curio
-	$(GOCC) build $(GOFLAGS) -o curio -ldflags " \
+	GOAMD64=v4 $(GOCC) build $(GOFLAGS) -o curio -ldflags " -s -w \
 	-X github.com/filecoin-project/curio/build.IsOpencl=$(FFI_USE_OPENCL) \
 	-X github.com/filecoin-project/curio/build.CurrentCommit=+git_`git log -1 --format=%h_%cI`" \
 	./cmd/curio
@@ -173,6 +175,7 @@ gen: gensimple
 
 gensimple: go-generate cfgdoc-gen api-gen docsgen docsgen-cli
 	$(GOCC) run ./scripts/fiximports
+	go mod tidy
 .PHONY: gen
 
 forest-test: GOFLAGS+=-tags=forest

alertmanager/alerts.go

@@ -16,9 +16,9 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 
+	"github.com/filecoin-project/curio/build"
 	"github.com/filecoin-project/curio/deps/config"
 
-	lbuild "github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 )
 
@@ -336,7 +336,7 @@ func wdPostCheck(al *alerts) {
 		return
 	}
 
-	from := head.Height() - abi.ChainEpoch(math.Ceil(AlertMangerInterval.Seconds()/float64(lbuild.BlockDelaySecs))) - 1
+	from := head.Height() - abi.ChainEpoch(math.Ceil(AlertMangerInterval.Seconds()/float64(build.BlockDelaySecs))) - 1
 	if from < 0 {
 		from = 0
 	}
@@ -468,7 +468,7 @@ func wnPostCheck(al *alerts) {
 		return
 	}
 
-	from := head.Height() - abi.ChainEpoch(math.Ceil(AlertMangerInterval.Seconds()/float64(lbuild.BlockDelaySecs))) - 1
+	from := head.Height() - abi.ChainEpoch(math.Ceil(AlertMangerInterval.Seconds()/float64(build.BlockDelaySecs))) - 1
 	if from < 0 {
 		from = 0
 	}
@@ -504,7 +504,7 @@ func wnPostCheck(al *alerts) {
 		return
 	}
 
-	epochs := int64(math.Ceil(AlertMangerInterval.Seconds() / float64(lbuild.BlockDelaySecs)))
+	epochs := int64(math.Ceil(AlertMangerInterval.Seconds() / float64(build.BlockDelaySecs)))
 	if (head.Height() - abi.ChainEpoch(epochs)) < 0 {
 		epochs = int64(head.Height())
 	}

alertmanager/curioalerting/al.go

@@ -0,0 +1,47 @@
+package curioalerting
+
+import (
+	"sync"
+
+	logging "github.com/ipfs/go-log/v2"
+
+	"github.com/filecoin-project/curio/lib/paths/alertinginterface"
+)
+
+var log = logging.Logger("curio/alerting")
+
+type AlertingSystem struct {
+	sync.Mutex
+	Current map[alertinginterface.AlertType]map[string]any
+}
+
+func NewAlertingSystem() *AlertingSystem {
+	return &AlertingSystem{Current: map[alertinginterface.AlertType]map[string]any{}}
+}
+
+func (as *AlertingSystem) AddAlertType(name, id string) alertinginterface.AlertType {
+	return alertinginterface.AlertType{
+		System:    id,
+		Subsystem: name,
+	}
+}
+
+func (as *AlertingSystem) Raise(alert alertinginterface.AlertType, metadata map[string]interface{}) {
+	as.Lock()
+	defer as.Unlock()
+	as.Current[alert] = metadata
+	log.Errorw("Alert raised: ", "System", alert.System, "Subsystem", alert.Subsystem, "Metadata", metadata)
+}
+
+func (as *AlertingSystem) IsRaised(alert alertinginterface.AlertType) bool {
+	as.Lock()
+	defer as.Unlock()
+	_, ok := as.Current[alert]
+	return ok
+}
+
+func (as *AlertingSystem) Resolve(alert alertinginterface.AlertType, metadata map[string]string) {
+	as.Lock()
+	defer as.Unlock()
+	delete(as.Current, alert)
+}

alertmanager/task_alert.go

@@ -13,6 +13,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/dline"
 
+	"github.com/filecoin-project/curio/alertmanager/curioalerting"
 	"github.com/filecoin-project/curio/alertmanager/plugin"
 	"github.com/filecoin-project/curio/deps/config"
 	"github.com/filecoin-project/curio/harmony/harmonydb"
@@ -42,6 +43,7 @@ type AlertTask struct {
 	cfg     config.CurioAlertingConfig
 	db      *harmonydb.DB
 	plugins []plugin.Plugin
+	al      *curioalerting.AlertingSystem
 }
 
 type alertOut struct {
@@ -67,12 +69,13 @@ var alertFuncs = []alertFunc{
 	wnPostCheck,
 }
 
-func NewAlertTask(api AlertAPI, db *harmonydb.DB, alertingCfg config.CurioAlertingConfig) *AlertTask {
+func NewAlertTask(
+	api AlertAPI, db *harmonydb.DB, alertingCfg config.CurioAlertingConfig, al *curioalerting.AlertingSystem) *AlertTask {
 	return &AlertTask{
-		api:     api,
-		db:      db,
-		cfg:     alertingCfg,
-		plugins: plugin.LoadAlertPlugins(alertingCfg),
+		api: api,
+		db:  db,
+		cfg: alertingCfg,
+		al:  al,
 	}
 }
 
@@ -111,6 +114,13 @@ func (a *AlertTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done
 			}
 		}
 	}
+	{
+		a.al.Lock()
+		defer a.al.Unlock()
+		for sys, meta := range a.al.Current {
+			details[sys.System+"_"+sys.Subsystem] = meta
+		}
+	}
 
 	// Alert only if required
 	if len(details) > 0 {

api/api_chain.go

@@ -3,11 +3,13 @@ package api
 import (
 	"context"
 
+	"github.com/google/uuid"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
+	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
@@ -18,14 +20,25 @@ import (
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
 // CurioChainRPC is a subset of the Filecoin API that is supported by Forest.
+// Complete list is available at github.com/orgs/ChainSafe/projects/29/views/1
 type CurioChainRPC interface {
-	Version(p0 context.Context) (api.APIVersion, error)
+	// ...
+	AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
+	AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error)    //perm:admin
+	Version(context.Context) (api.APIVersion, error)                         //perm:read
+	Shutdown(context.Context) error                                          //perm:admin
+	Session(context.Context) (uuid.UUID, error)                              //perm:read
+
+	// Chain
 	ChainHead(context.Context) (*types.TipSet, error)
 	ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
 	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error)
+	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) //perm:read
+	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)                                 //perm:read
 	StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
 	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
@@ -68,12 +81,23 @@ type CurioChainRPC interface {
 	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
 	ChainPutObj(context.Context, blocks.Block) error
-
-	// Added afterward
 	MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error)
 	StateMinerActiveSectors(ctx context.Context, maddr address.Address, tsk types.TipSetKey) ([]*miner.SectorOnChainInfo, error)
 	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorLocation, error)
-	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (big.Int, error)
+
+	StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error)
+	StateListMessages(context.Context, *api.MessageMatch, types.TipSetKey, abi.ChainEpoch) ([]cid.Cid, error)
+	StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error)
+	StateMarketBalance(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error)
+	StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error)
+	StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)
+	StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)
+	StateNetworkName(context.Context) (dtypes.NetworkName, error)
+	StateReadState(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error)
+	StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error)
+	StateVerifiedClientStatus(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error)
+	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error)
+	StateCirculatingSupply(context.Context, types.TipSetKey) (big.Int, error)
 }
 
 var _ CurioChainRPC = api.FullNode(nil)

api/api_curio.go

@@ -18,8 +18,8 @@ type Curio interface {
 	// MethodGroup: Curio
 	//The common method group contains administration methods
 
-	Version(context.Context) (api.Version, error) //perm:admin
-	Shutdown(context.Context) error               //perm:admin
+	Version(context.Context) ([]int, error) //perm:admin
+	Shutdown(context.Context) error         //perm:admin
 
 	// MethodGroup: Deal
 	//The deal method group contains method for adding deals to sector

api/client/client.go

@@ -7,13 +7,11 @@ import (
 	"github.com/filecoin-project/go-jsonrpc"
 
 	"github.com/filecoin-project/curio/api"
-
-	"github.com/filecoin-project/lotus/api/v1api"
 )
 
 // NewCurioRpc creates a new http jsonrpc client.
 func NewCurioRpc(ctx context.Context, addr string, requestHeader http.Header) (api.Curio, jsonrpc.ClientCloser, error) {
-	var res v1api.CurioStruct
+	var res api.CurioStruct
 
 	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		api.GetInternalStructs(&res), requestHeader, jsonrpc.WithErrors(RPCErrors))