lightning: Compress Reader/Writer supports reading/writing Snappy/Zstd type compressed files #38603
Conversation
|
[REVIEW NOTIFICATION] This pull request has been approved by:
To complete the pull request process, please ask the reviewers in the list to review by filling The full list of commands accepted by this bot can be found here. Reviewer can indicate their review by submitting an approval review. |
ti-chi-bot
added
release-note-none
|
/cc @lichunzhu @lance6716 |
|
BTW, Have we a plan to support zstd ? |
Thanks for the suggestion. I will try to implement supporting zstd in the later PR if possible. |
|
snappy part LGTM. Looking forward to lzo/zstd. |
Okay, in progress |
br/pkg/storage/writer_test.go
Outdated
| testFnSnappy := func(test *testcase, t *testing.T) { | ||
| t.Log(test.name) | ||
| backend, err := ParseBackend("local://"+filepath.ToSlash(dir), nil) | ||
| require.NoError(t, err) | ||
| ctx := context.Background() | ||
| storage, err := Create(ctx, backend, true) | ||
| require.NoError(t, err) | ||
| storage = WithCompression(storage, Snappy) | ||
| fileName := strings.ReplaceAll(test.name, " ", "-") + ".txt.snappy" | ||
| writer, err := storage.Create(ctx, fileName) | ||
| require.NoError(t, err) | ||
| for _, str := range test.content { | ||
| p := []byte(str) | ||
| written, err2 := writer.Write(ctx, p) | ||
| require.Nil(t, err2) | ||
| require.Len(t, p, written) | ||
| } | ||
| err = writer.Close(ctx) | ||
| require.NoError(t, err) | ||
|
|
||
| // make sure compressed file is written correctly | ||
| file, err := os.Open(filepath.Join(dir, fileName)) | ||
| require.NoError(t, err) | ||
| r, err := newCompressReader(test.compressType, file) | ||
| require.NoError(t, err) | ||
| var bf bytes.Buffer | ||
| _, err = bf.ReadFrom(r) | ||
| require.NoError(t, err) | ||
| require.Equal(t, strings.Join(test.content, ""), bf.String()) | ||
|
|
||
| // test withCompression Open | ||
| r, err = storage.Open(ctx, fileName) | ||
| require.NoError(t, err) | ||
| content, err := io.ReadAll(r) | ||
| require.NoError(t, err) | ||
| require.Equal(t, strings.Join(test.content, ""), string(content)) | ||
|
|
||
| require.Nil(t, file.Close()) | ||
| } | ||
| compressTypeSnappyArr := []CompressType{Snappy} |
There was a problem hiding this comment.
Can we combine these two functions?
|
/run-all-tests |
br/pkg/storage/writer_test.go
Outdated
| fileName := strings.ReplaceAll(test.name, " ", "-") + ".txt.gz" | ||
| storage = WithCompression(storage, test.compressType) | ||
| var suffix string | ||
| if test.compressType == Gzip { |
There was a problem hiding this comment.
How about implementing a String() function for these compressType?
|
zstd is supported now. |
lyzx2001
changed the title
|
/run-all-tests |
ti-chi-bot
added
status/LGT2
|
/merge |
|
This pull request has been accepted and is ready to merge. Commit hash: 215831a
|
ti-chi-bot
added
the
status/can-merge
TiDB MergeCI notify✅ Well Done! New fixed [1] after this pr merged.
|
What problem does this PR solve?
Issue Number: ref #38514
Problem Summary:
What is changed and how it works?
Compress Reader supports reading Snappy/Zstd type compressed files
Compress Writer supports writing Snappy/Zstd type compressed files
Check List
Tests
Side effects
Documentation
Release note
Please refer to Release Notes Language Style Guide to write a quality release note.