This commit is contained in:
parent
d2205b11a7
commit
02b9aebbe5
341 changed files with 1571 additions and 32574 deletions
|
|
@ -4,7 +4,7 @@ import (
|
|||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/TecharoHQ/anubis/internal/actorify"
|
||||
"git.sad.ovh/sophie/nuke/internal/actorify"
|
||||
)
|
||||
|
||||
type unit struct{}
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@
|
|||
package all
|
||||
|
||||
import (
|
||||
_ "github.com/TecharoHQ/anubis/lib/store/bbolt"
|
||||
_ "github.com/TecharoHQ/anubis/lib/store/memory"
|
||||
_ "github.com/TecharoHQ/anubis/lib/store/s3api"
|
||||
_ "github.com/TecharoHQ/anubis/lib/store/valkey"
|
||||
_ "git.sad.ovh/sophie/nuke/lib/store/bbolt"
|
||||
_ "git.sad.ovh/sophie/nuke/lib/store/memory"
|
||||
_ "git.sad.ovh/sophie/nuke/lib/store/s3api"
|
||||
_ "git.sad.ovh/sophie/nuke/lib/store/valkey"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import (
|
|||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/TecharoHQ/anubis/lib/store"
|
||||
"git.sad.ovh/sophie/nuke/lib/store"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
|
|
@ -20,17 +20,17 @@ var (
|
|||
//
|
||||
// In essence, bbolt is a hierarchical key/value store with a twist: every value
|
||||
// needs to belong to a bucket. Buckets can contain an infinite number of
|
||||
// buckets. As such, Anubis nests values in buckets. Each value in the store
|
||||
// buckets. As such, Nuke nests values in buckets. Each value in the store
|
||||
// is given its own bucket with two keys:
|
||||
//
|
||||
// 1. data - The raw data, usually in JSON
|
||||
// 2. expiry - The expiry time formatted as a time.RFC3339Nano timestamp string
|
||||
//
|
||||
// When Anubis stores a new bit of data, it creates a new bucket for that value.
|
||||
// When Nuke stores a new bit of data, it creates a new bucket for that value.
|
||||
// This allows the cleanup phase to iterate over every bucket in the database and
|
||||
// only scan the expiry times without having to decode the entire record.
|
||||
//
|
||||
// bbolt is not suitable for environments where multiple instance of Anubis need
|
||||
// bbolt is not suitable for environments where multiple instance of Nuke need
|
||||
// to read from and write to the same backend store. For that, use the valkey
|
||||
// storage backend.
|
||||
//
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import (
|
|||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/TecharoHQ/anubis/lib/store/storetest"
|
||||
"git.sad.ovh/sophie/nuke/lib/store/storetest"
|
||||
)
|
||||
|
||||
func TestImpl(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/TecharoHQ/anubis/lib/store"
|
||||
"git.sad.ovh/sophie/nuke/lib/store"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
|
|
@ -68,7 +68,7 @@ func (Factory) Valid(data json.RawMessage) error {
|
|||
|
||||
// Config is the bbolt storage backend configuration.
|
||||
type Config struct {
|
||||
// Path is the filesystem path of the database. The folder must be writable to Anubis.
|
||||
// Path is the filesystem path of the database. The folder must be writable to Nuke.
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ var (
|
|||
ErrBadConfig = errors.New("store: configuration is invalid")
|
||||
)
|
||||
|
||||
// Interface defines the calls that Anubis uses for storage in a local or remote
|
||||
// Interface defines the calls that Nuke uses for storage in a local or remote
|
||||
// datastore. This can be implemented with an in-memory, on-disk, or in-database
|
||||
// storage backend.
|
||||
type Interface interface {
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/TecharoHQ/anubis/lib/store"
|
||||
"github.com/TecharoHQ/anubis/lib/store/memory"
|
||||
"git.sad.ovh/sophie/nuke/lib/store"
|
||||
"git.sad.ovh/sophie/nuke/lib/store/memory"
|
||||
)
|
||||
|
||||
func TestJSON(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/TecharoHQ/anubis/decaymap"
|
||||
"github.com/TecharoHQ/anubis/lib/store"
|
||||
"git.sad.ovh/sophie/nuke/decaymap"
|
||||
"git.sad.ovh/sophie/nuke/lib/store"
|
||||
)
|
||||
|
||||
type factory struct{}
|
||||
|
|
@ -66,7 +66,7 @@ func (i *impl) cleanupThread(ctx context.Context) {
|
|||
}
|
||||
}
|
||||
|
||||
// New creates a simple in-memory store. This will not scale to multiple Anubis instances.
|
||||
// New creates a simple in-memory store. This will not scale to multiple Nuke instances.
|
||||
func New(ctx context.Context) store.Interface {
|
||||
result := &impl{
|
||||
store: decaymap.New[string, []byte](),
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package memory
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/TecharoHQ/anubis/lib/store/storetest"
|
||||
"git.sad.ovh/sophie/nuke/lib/store/storetest"
|
||||
)
|
||||
|
||||
func TestImpl(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/TecharoHQ/anubis/lib/store"
|
||||
"git.sad.ovh/sophie/nuke/lib/store"
|
||||
awsConfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/TecharoHQ/anubis/lib/store"
|
||||
"git.sad.ovh/sophie/nuke/lib/store"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
|
|
@ -40,7 +40,7 @@ func (s *Store) Get(ctx context.Context, key string) ([]byte, error) {
|
|||
return nil, fmt.Errorf("%w: %w", store.ErrNotFound, err)
|
||||
}
|
||||
defer out.Body.Close()
|
||||
if msStr, ok := out.Metadata["x-anubis-expiry-ms"]; ok && msStr != "" {
|
||||
if msStr, ok := out.Metadata["x-nuke-expiry-ms"]; ok && msStr != "" {
|
||||
if ms, err := strconv.ParseInt(msStr, 10, 64); err == nil {
|
||||
if time.Now().UnixMilli() >= ms {
|
||||
_, _ = s.s3.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &s.bucket, Key: &normKey})
|
||||
|
|
@ -57,11 +57,11 @@ func (s *Store) Get(ctx context.Context, key string) ([]byte, error) {
|
|||
|
||||
func (s *Store) Set(ctx context.Context, key string, value []byte, expiry time.Duration) error {
|
||||
normKey := strings.ReplaceAll(key, ":", "/")
|
||||
// S3 has no native TTL; we store object with metadata X-Anubis-Expiry as epoch seconds.
|
||||
// S3 has no native TTL; we store object with metadata X-Nuke-Expiry as epoch seconds.
|
||||
var meta map[string]string
|
||||
if expiry > 0 {
|
||||
exp := time.Now().Add(expiry).UnixMilli()
|
||||
meta = map[string]string{"x-anubis-expiry-ms": fmt.Sprintf("%d", exp)}
|
||||
meta = map[string]string{"x-nuke-expiry-ms": fmt.Sprintf("%d", exp)}
|
||||
}
|
||||
_, err := s.s3.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &s.bucket,
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/TecharoHQ/anubis/lib/store/storetest"
|
||||
"git.sad.ovh/sophie/nuke/lib/store/storetest"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
|
@ -91,7 +91,7 @@ func TestKeyNormalization(t *testing.T) {
|
|||
f := Factory{Client: mock}
|
||||
|
||||
data, _ := json.Marshal(Config{
|
||||
BucketName: "anubis",
|
||||
BucketName: "nuke",
|
||||
})
|
||||
|
||||
s, err := f.Build(t.Context(), json.RawMessage(data))
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/TecharoHQ/anubis/lib/store"
|
||||
"git.sad.ovh/sophie/nuke/lib/store"
|
||||
)
|
||||
|
||||
func Common(t *testing.T, f store.Factory, config json.RawMessage) {
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/TecharoHQ/anubis/internal"
|
||||
"github.com/TecharoHQ/anubis/lib/store"
|
||||
"git.sad.ovh/sophie/nuke/internal"
|
||||
"git.sad.ovh/sophie/nuke/lib/store"
|
||||
valkey "github.com/redis/go-redis/v9"
|
||||
"github.com/redis/go-redis/v9/maintnotifications"
|
||||
)
|
||||
|
|
@ -27,7 +27,7 @@ var (
|
|||
ErrSentinelAddrEmpty = errors.New("valkey.Sentinel: addr cannot be empty")
|
||||
)
|
||||
|
||||
// Config is what Anubis unmarshals from the "parameters" JSON.
|
||||
// Config is what Nuke unmarshals from the "parameters" JSON.
|
||||
type Config struct {
|
||||
URL string `json:"url"`
|
||||
Cluster bool `json:"cluster,omitempty"`
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/TecharoHQ/anubis/lib/store"
|
||||
"git.sad.ovh/sophie/nuke/lib/store"
|
||||
valkey "github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
|
|
@ -41,7 +41,7 @@ func (s *Store) Delete(ctx context.Context, key string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// IsPersistent tells Anubis this backend is “real” storage, not in-memory.
|
||||
// IsPersistent tells Nuke this backend is “real” storage, not in-memory.
|
||||
func (s *Store) IsPersistent() bool {
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import (
|
|||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/TecharoHQ/anubis/lib/store/storetest"
|
||||
"git.sad.ovh/sophie/nuke/lib/store/storetest"
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue