Merge pull request #33 from Photoroom/ben/fix_CI_perf

[perf] Tentative perf fix for the pypi package (local builds were fast already)
Photoroom · Nov 13, 2024 · e804dbb · e804dbb
2 parents 6531f57 + c4f49b2
commit e804dbb
Show file tree

Hide file tree

Showing 8 changed files with 19 additions and 12 deletions.
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
@@ -47,7 +47,7 @@ jobs:
       - name: Build python module
         run: |
           cd pkg
-          gopy pkg -author="Photoroom" -email="[email protected]" -name="datago" -version="${{ steps.version.outputs.major-version }}.${{ steps.version.outputs.minor-version }}.${{ steps.version.outputs.patch-version }}" .
+          GOAMD64=v3 gopy pkg -author="Photoroom" -email="[email protected]" -name="datago" -version="${{ steps.version.outputs.major-version }}.${{ steps.version.outputs.minor-version }}.${{ steps.version.outputs.patch-version }}" .
           export DESTINATION="../build"
           mkdir -p $DESTINATION/datago
           mv datago/* $DESTINATION/datago/.

diff --git a/generate_python_package.sh b/generate_python_package.sh
@@ -12,7 +12,7 @@ rm -rf $DESTINATION
 
 # Build the python package via the gopy toolchain
 cd pkg
-gopy pkg -author="Photoroom" -email="[email protected]" -url="" -name="datago" -version="0.0.1" .
+GOAMD64=v3 gopy pkg -author="Photoroom" -email="[email protected]" -url="" -name="datago" -version="0.0.1" .
 mkdir -p $DESTINATION/datago
 mv datago/* $DESTINATION/datago/.
 mv setup.py $DESTINATION/.

diff --git a/go.mod b/go.mod
@@ -7,6 +7,7 @@ require golang.org/x/image v0.18.0 // indirect
 require (
 	github.com/davidbyttow/govips/v2 v2.15.0
 	github.com/go-python/gopy v0.4.10
+	go.uber.org/automaxprocs v1.6.0
 )
 
 require (

diff --git a/go.sum b/go.sum
@@ -13,7 +13,10 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
+go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
@@ -67,3 +70,4 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/pkg/client.go b/pkg/client.go
@@ -12,6 +12,7 @@ import (
 	"sync"
 
 	"github.com/davidbyttow/govips/v2/vips"
+	_ "go.uber.org/automaxprocs/maxprocs"
 )
 
 // -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -278,7 +279,7 @@ func (c *DatagoClient) Start() {
 // Get a deserialized sample from the client
 func (c *DatagoClient) GetSample() Sample {
 	if c.cancel == nil && c.servedSamples == 0 {
-		fmt.Println("Dataroom client not started. Starting it on the first sample, this adds some initial latency")
+		fmt.Println("Datago client not started. Starting it on the first sample, this adds some initial latency")
 		fmt.Println("Please consider starting the client in anticipation by calling .Start()")
 		c.Start()
 	}
@@ -300,7 +301,7 @@ func (c *DatagoClient) GetSample() Sample {
 
 // Stop the background downloads, will clear the memory and CPU footprint
 func (c *DatagoClient) Stop() {
-	fmt.Println("Stopping the dataroom client")
+	fmt.Println("Stopping the datago client")
 
 	// Signal the coroutines that next round should be a stop
 	if c.cancel == nil {
@@ -318,7 +319,7 @@ func (c *DatagoClient) Stop() {
 		c.waitGroup.Wait()
 	}
 
-	fmt.Println("Dataroom client stopped")
+	fmt.Println("Datago client stopped")
 	c.cancel = nil
 	c.context = nil
 }
@@ -332,7 +333,6 @@ func (c *DatagoClient) asyncDispatch() {
 	for {
 		select {
 		case <-c.context.Done():
-			fmt.Println("Metadata fetch goroutine wrapping up")
 			close(c.chanSampleMetadata)
 			return
 		case page, open := <-c.chanPages:

diff --git a/pkg/worker_filesystem.go b/pkg/worker_filesystem.go
@@ -64,5 +64,4 @@ func (b BackendFileSystem) collectSamples(chanSampleMetadata chan SampleDataPoin
 		<-ack_channel
 	}
 	close(chanSamples)
-	fmt.Println("No more items to serve, wrapping up")
 }
diff --git a/pkg/worker_http.go b/pkg/worker_http.go
@@ -1,7 +1,6 @@
 package datago
 
 import (
-	"fmt"
 	"net/http"
 	"time"
 )
@@ -49,5 +48,4 @@ func (b BackendHTTP) collectSamples(chanSampleMetadata chan SampleDataPointers,
 		<-ack_channel
 	}
 	close(chanSamples)
-	fmt.Println("No more items to serve, wrapping up")
 }
diff --git a/python/benchmark_filesystem.py b/python/benchmark_filesystem.py
@@ -6,6 +6,7 @@
 import json
 import typer
 
+
 def benchmark(
     root_path: str = typer.Option(
         os.getenv("DATAROOM_TEST_FILESYSTEM", ""), help="The source to test out"
@@ -34,7 +35,7 @@ def benchmark(
             "max_aspect_ratio": 2.0,
             "pre_encode_images": False,
         },
-        "prefetch_buffer_size": 64,
+        "prefetch_buffer_size": 256,
         "samples_buffer_size": 128,
         "concurrency": concurrency,
         "limit": limit,
@@ -76,15 +77,19 @@ def benchmark(
         transform = (
             transforms.Compose(
                 [
-                    transforms.Resize((512, 512), interpolation=transforms.InterpolationMode.BICUBIC),
+                    transforms.Resize(
+                        (512, 512), interpolation=transforms.InterpolationMode.BICUBIC
+                    ),
                 ]
             )
             if crop_and_resize
             else None
         )
 
         # Create the ImageFolder dataset
-        dataset = datasets.ImageFolder(root=root_path, transform=transform, allow_empty=True)
+        dataset = datasets.ImageFolder(
+            root=root_path, transform=transform, allow_empty=True
+        )
 
         # Create a DataLoader to allow for multiple workers
         dataloader = DataLoader(