From c35b47447f1bdbe7ecb23f5d62bef8d5da66b350 Mon Sep 17 00:00:00 2001 From: itzaname Date: Sun, 22 Jun 2025 12:10:52 -0400 Subject: [PATCH] Add rpc --- cmd/dev-service/service.go | 3 +- go.mod | 5 +- go.sum | 8 + pkg/api/middleware/validate.go | 59 ++++++++ pkg/cache/cache.go | 81 ++++++++++ pkg/cmds/root.go | 42 ++++++ pkg/cmds/rpc.go | 75 ++++++++++ pkg/cmds/{serve.go => web.go} | 52 +------ pkg/datastore/datastore.go | 1 + pkg/datastore/gormstore/application.go | 15 ++ pkg/datastore/permission.go | 2 +- pkg/model/application.go | 9 ++ pkg/model/permission.go | 1 - pkg/ratelimit/lua.go | 92 ++++++++++++ pkg/ratelimit/rate_limit.go | 124 ++++++++++++++++ pkg/rpc/dev.go | 195 +++++++++++++++++++++++++ 16 files changed, 713 insertions(+), 51 deletions(-) create mode 100644 pkg/api/middleware/validate.go create mode 100644 pkg/cache/cache.go create mode 100644 pkg/cmds/rpc.go rename pkg/cmds/{serve.go => web.go} (52%) create mode 100644 pkg/ratelimit/lua.go create mode 100644 pkg/ratelimit/rate_limit.go create mode 100644 pkg/rpc/dev.go diff --git a/cmd/dev-service/service.go b/cmd/dev-service/service.go index baf9a45..4b70760 100644 --- a/cmd/dev-service/service.go +++ b/cmd/dev-service/service.go @@ -16,7 +16,8 @@ func main() { app := cmds.NewApp() app.Commands = []*cli.Command{ - cmds.NewServeCommand(), + cmds.NewWebCommand(), + cmds.NewRpcCommand(), } if err := app.Run(os.Args); err != nil { diff --git a/go.mod b/go.mod index 79c83a8..511c72d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.itzana.me/strafesnet/dev-portal go 1.24.0 require ( - git.itzana.me/strafesnet/go-grpc v0.0.0-20250622012600-01ac9fdf02d8 + git.itzana.me/strafesnet/go-grpc v0.0.0-20250622145054-0c0eb0ba26c7 git.itzana.me/strafesnet/utils v0.0.0-20220716194944-d8ca164052f9 github.com/gin-gonic/gin v1.10.0 github.com/sirupsen/logrus v1.9.3 @@ -16,9 +16,11 @@ require ( require ( github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/elazarl/go-bindata-assetfs v1.0.1 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -42,6 +44,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/redis/go-redis/v9 v9.10.0 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/go.sum b/go.sum index 93c4e8b..a2773cc 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= git.itzana.me/strafesnet/go-grpc v0.0.0-20250622012600-01ac9fdf02d8 h1:QHlKFYqiNb9Rp5mqDMlQ05uVsVs5p52rJhqtwfybVrg= git.itzana.me/strafesnet/go-grpc v0.0.0-20250622012600-01ac9fdf02d8/go.mod h1:X7XTRUScRkBWq8q8bplbeso105RPDlnY7J6Wy1IwBMs= +git.itzana.me/strafesnet/go-grpc v0.0.0-20250622145054-0c0eb0ba26c7 h1:k6Skqr00NOo9Do9Z5rxqzRSR+1BR/bY93+Lf86QlFV8= +git.itzana.me/strafesnet/go-grpc v0.0.0-20250622145054-0c0eb0ba26c7/go.mod h1:X7XTRUScRkBWq8q8bplbeso105RPDlnY7J6Wy1IwBMs= git.itzana.me/strafesnet/utils v0.0.0-20220716194944-d8ca164052f9 h1:7lU6jyR7S7Rhh1dnUp7GyIRHUTBXZagw8F4n4hOyxLw= git.itzana.me/strafesnet/utils v0.0.0-20220716194944-d8ca164052f9/go.mod h1:uyYerSieEt4v0MJCdPLppG0LtJ4Yj035vuTetWGsxjY= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -12,6 +14,8 @@ github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3z github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= @@ -29,6 +33,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -121,6 +127,8 @@ github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/redis/go-redis/v9 v9.10.0 h1:FxwK3eV8p/CQa0Ch276C7u2d0eNC9kCmAYQ7mCXCzVs= +github.com/redis/go-redis/v9 v9.10.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= diff --git a/pkg/api/middleware/validate.go b/pkg/api/middleware/validate.go new file mode 100644 index 0000000..7ac4187 --- /dev/null +++ b/pkg/api/middleware/validate.go @@ -0,0 +1,59 @@ +package middleware + +import ( + "git.itzana.me/strafesnet/go-grpc/dev" + "github.com/gin-gonic/gin" + "google.golang.org/grpc" + "net/http" + "strconv" +) + +const ( + HeaderAPIKey = "X-API-Key" + HeaderRateLimitBurst = "X-Rate-Limit-Burst" + HeaderRateLimitDaily = "X-Rate-Limit-Daily" + HeaderRateLimitMonthly = "X-Rate-Limit-Monthly" + + // Error messages + ErrMsgAPIKeyRequired = "API key is required" + ErrMsgInternalServer = "Internal server error" +) + +// ValidateRequest returns a middleware that validates API requests against the dev service +func ValidateRequest(service, permission string, conn *grpc.ClientConn) gin.HandlerFunc { + return func(ctx *gin.Context) { + // Get API key from the request header + apiKey := ctx.GetHeader(HeaderAPIKey) + if apiKey == "" { + ctx.JSON(http.StatusUnauthorized, gin.H{"error": ErrMsgAPIKeyRequired}) + ctx.Abort() + return + } + + // Create client and validate the API key + resp, err := dev.NewDevServiceClient(conn).Validate(ctx, &dev.APIValidationRequest{ + Service: service, + Permission: permission, + Key: apiKey, + }) + + if err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{"error": ErrMsgInternalServer}) + ctx.Abort() + return + } + + if !resp.Valid { + ctx.JSON(http.StatusUnauthorized, gin.H{"error": resp.ErrorMessage}) + ctx.Abort() + return + } + + // Set the remaining rate limits in the response headers + ctx.Header(HeaderRateLimitBurst, strconv.FormatUint(resp.RemainingBurst, 10)) + ctx.Header(HeaderRateLimitDaily, strconv.FormatUint(resp.RemainingDaily, 10)) + ctx.Header(HeaderRateLimitMonthly, strconv.FormatUint(resp.RemainingMonthly, 10)) + + ctx.Next() + } +} diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go new file mode 100644 index 0000000..b621039 --- /dev/null +++ b/pkg/cache/cache.go @@ -0,0 +1,81 @@ +package cache + +import ( + "context" + "encoding/json" + "fmt" + "github.com/redis/go-redis/v9" + "time" + + "git.itzana.me/strafesnet/dev-portal/pkg/model" +) + +// Cache is a simple caching interface for Redis +type Cache struct { + client *redis.Client + prefix string +} + +// NewCache creates a new Redis cache client +func NewCache(client *redis.Client, prefix string) *Cache { + return &Cache{ + client: client, + prefix: prefix, + } +} + +// GetApplicationByAPIKey retrieves an application from cache by API key +func (c *Cache) GetApplicationByAPIKey(ctx context.Context, apiKey string) (*model.ApplicationCache, error) { + key := c.getAppAPIKeyCache(apiKey) + + val, err := c.client.Get(ctx, key).Result() + if err != nil { + if err == redis.Nil { + return nil, fmt.Errorf("application with API key %s not found in cache", apiKey) + } + return nil, fmt.Errorf("failed to get from cache: %w", err) + } + + var app model.ApplicationCache + if err := json.Unmarshal([]byte(val), &app); err != nil { + return nil, fmt.Errorf("failed to unmarshal application: %w", err) + } + + return &app, nil +} + +// SetApplicationByAPIKey caches an application with the given API key and expiration +func (c *Cache) SetApplicationByAPIKey(ctx context.Context, app *model.ApplicationCache, expiration time.Duration) error { + if app == nil || app.APIKey == "" { + return fmt.Errorf("invalid application or missing API key") + } + + key := c.getAppAPIKeyCache(app.APIKey) + + data, err := json.Marshal(app) + if err != nil { + return fmt.Errorf("failed to marshal application: %w", err) + } + + if err := c.client.Set(ctx, key, data, expiration).Err(); err != nil { + return fmt.Errorf("failed to set in cache: %w", err) + } + + return nil +} + +// DeleteApplicationByAPIKey removes an application from cache by API key +func (c *Cache) DeleteApplicationByAPIKey(ctx context.Context, apiKey string) error { + key := c.getAppAPIKeyCache(apiKey) + + if err := c.client.Del(ctx, key).Err(); err != nil { + return fmt.Errorf("failed to delete from cache: %w", err) + } + + return nil +} + +// getAppAPIKeyCache generates the cache key for an application by API key +func (c *Cache) getAppAPIKeyCache(apiKey string) string { + return fmt.Sprintf("%s:app:apikey:%s", c.prefix, apiKey) +} diff --git a/pkg/cmds/root.go b/pkg/cmds/root.go index 836b476..52d1c12 100644 --- a/pkg/cmds/root.go +++ b/pkg/cmds/root.go @@ -24,6 +24,48 @@ var ( return nil }, }, + &cli.StringFlag{ + Name: "pg-host", + Usage: "Host of postgres database", + EnvVars: []string{"PG_HOST"}, + Required: true, + }, + &cli.IntFlag{ + Name: "pg-port", + Usage: "Port of postgres database", + EnvVars: []string{"PG_PORT"}, + Required: true, + }, + &cli.StringFlag{ + Name: "pg-db", + Usage: "Name of database to connect to", + EnvVars: []string{"PG_DB"}, + Required: true, + }, + &cli.StringFlag{ + Name: "pg-user", + Usage: "User to connect with", + EnvVars: []string{"PG_USER"}, + Required: true, + }, + &cli.StringFlag{ + Name: "pg-password", + Usage: "Password to connect with", + EnvVars: []string{"PG_PASSWORD"}, + Required: true, + }, + &cli.BoolFlag{ + Name: "migrate", + Usage: "Run database migrations", + Value: true, + EnvVars: []string{"MIGRATE"}, + }, + &cli.IntFlag{ + Name: "port", + Usage: "Port to listen on", + Value: 8080, + EnvVars: []string{"PORT"}, + }, } ) diff --git a/pkg/cmds/rpc.go b/pkg/cmds/rpc.go new file mode 100644 index 0000000..b5a82b1 --- /dev/null +++ b/pkg/cmds/rpc.go @@ -0,0 +1,75 @@ +package cmds + +import ( + "fmt" + "git.itzana.me/strafesnet/dev-portal/pkg/cache" + "git.itzana.me/strafesnet/dev-portal/pkg/datastore/gormstore" + "git.itzana.me/strafesnet/dev-portal/pkg/ratelimit" + "git.itzana.me/strafesnet/dev-portal/pkg/rpc" + "git.itzana.me/strafesnet/go-grpc/dev" + "github.com/redis/go-redis/v9" + "github.com/urfave/cli/v2" + "google.golang.org/grpc" + "net" +) + +func NewRpcCommand() *cli.Command { + return &cli.Command{ + Name: "rpc", + Usage: "Run dev rpc service", + Action: rpcServer, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "redis-addr", + Usage: "Address of redis database", + EnvVars: []string{"REDIS_ADDR"}, + Required: true, + }, + &cli.StringFlag{ + Name: "redis-pass", + Usage: "Password of redis database", + EnvVars: []string{"REDIS_PASS"}, + Required: false, + Value: "", + }, + &cli.IntFlag{ + Name: "redis-db", + Usage: "Number of database to connect to", + EnvVars: []string{"REDIS_DB"}, + Required: true, + }, + }, + } +} + +func rpcServer(ctx *cli.Context) error { + // Setup database + store, err := gormstore.New(ctx) + if err != nil { + return err + } + + // Redis setup + rdb := redis.NewClient(&redis.Options{ + Addr: ctx.String("redis-addr"), + Password: ctx.String("redis-pass"), + DB: ctx.Int("redis-db"), + }) + + // grpc server + ls, err := net.Listen("tcp", fmt.Sprintf(":%d", ctx.Int("port"))) + if err != nil { + return err + } + + grpcServer := grpc.NewServer() + + dev.RegisterDevServiceServer(grpcServer, &rpc.Dev{ + Store: store, + Limiter: ratelimit.New(rdb), + Cache: cache.NewCache(rdb, "app"), + }) + + return grpcServer.Serve(ls) + +} diff --git a/pkg/cmds/serve.go b/pkg/cmds/web.go similarity index 52% rename from pkg/cmds/serve.go rename to pkg/cmds/web.go index ae9a208..f7b84f8 100644 --- a/pkg/cmds/serve.go +++ b/pkg/cmds/web.go @@ -10,54 +10,12 @@ import ( "google.golang.org/grpc/credentials/insecure" ) -func NewServeCommand() *cli.Command { +func NewWebCommand() *cli.Command { return &cli.Command{ - Name: "serve", - Usage: "Run dev service", - Action: rpcServer, + Name: "web", + Usage: "Run dev web service", + Action: runWeb, Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "pg-host", - Usage: "Host of postgres database", - EnvVars: []string{"PG_HOST"}, - Required: true, - }, - &cli.IntFlag{ - Name: "pg-port", - Usage: "Port of postgres database", - EnvVars: []string{"PG_PORT"}, - Required: true, - }, - &cli.StringFlag{ - Name: "pg-db", - Usage: "Name of database to connect to", - EnvVars: []string{"PG_DB"}, - Required: true, - }, - &cli.StringFlag{ - Name: "pg-user", - Usage: "User to connect with", - EnvVars: []string{"PG_USER"}, - Required: true, - }, - &cli.StringFlag{ - Name: "pg-password", - Usage: "Password to connect with", - EnvVars: []string{"PG_PASSWORD"}, - Required: true, - }, - &cli.BoolFlag{ - Name: "migrate", - Usage: "Run database migrations", - Value: true, - EnvVars: []string{"MIGRATE"}, - }, - &cli.IntFlag{ - Name: "port", - Usage: "Port to listen on", - Value: 8080, - EnvVars: []string{"PORT"}, - }, &cli.StringFlag{ Name: "auth-rpc-host", Usage: "Host of auth rpc", @@ -74,7 +32,7 @@ func NewServeCommand() *cli.Command { } } -func rpcServer(ctx *cli.Context) error { +func runWeb(ctx *cli.Context) error { // Setup database store, err := gormstore.New(ctx) if err != nil { diff --git a/pkg/datastore/datastore.go b/pkg/datastore/datastore.go index ec1f48e..603321c 100644 --- a/pkg/datastore/datastore.go +++ b/pkg/datastore/datastore.go @@ -41,6 +41,7 @@ type Datastore interface { // Application CreateApplication(ctx context.Context, app *model.Application) error GetApplication(ctx context.Context, id uint32) (*model.Application, error) + GetApplicationByAPIKey(ctx context.Context, apiKey string) (*model.Application, error) GetAllApplications(ctx context.Context) ([]model.Application, error) GetApplicationsForUser(ctx context.Context, userID uint64) ([]model.Application, error) UpdateApplication(ctx context.Context, app *model.Application) error diff --git a/pkg/datastore/gormstore/application.go b/pkg/datastore/gormstore/application.go index d85f51f..f73d124 100644 --- a/pkg/datastore/gormstore/application.go +++ b/pkg/datastore/gormstore/application.go @@ -24,6 +24,21 @@ func (g *Gormstore) GetApplication(ctx context.Context, id uint32) (*model.Appli return &application, result.Error } +// GetApplicationByAPIKey retrieves a single Application by its API key. +func (g *Gormstore) GetApplicationByAPIKey(ctx context.Context, apiKey string) (*model.Application, error) { + var application model.Application + result := g.db.WithContext(ctx). + Where("api_key = ?", apiKey). + Preload("User"). + Preload("Permissions"). + First(&application) + + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return nil, datastore.ErrNotExists + } + return &application, result.Error +} + // GetAllApplications retrieves all Applications from the database. func (g *Gormstore) GetAllApplications(ctx context.Context) ([]model.Application, error) { var applications []model.Application diff --git a/pkg/datastore/permission.go b/pkg/datastore/permission.go index d7b7003..b09ac90 100644 --- a/pkg/datastore/permission.go +++ b/pkg/datastore/permission.go @@ -24,7 +24,7 @@ func GetDefaultPermissionDefinitions() []model.Permission { PermissionName: "Read", Title: "Read Data", Description: "Provides access to view moderation logs", - IsDefault: true, + IsDefault: false, }, { Service: "Moderation", diff --git a/pkg/model/application.go b/pkg/model/application.go index bfc150f..1b7c359 100644 --- a/pkg/model/application.go +++ b/pkg/model/application.go @@ -17,3 +17,12 @@ type Application struct { CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } + +type ApplicationCache struct { + Name string `json:"name"` + UserID uint64 `json:"user_id"` + APIKey string `json:"api_key"` + Permissions []Permission `json:"permissions"` + Active bool `json:"active"` + RateLimit RateLimit `json:"rate_limit"` +} diff --git a/pkg/model/permission.go b/pkg/model/permission.go index c0080b0..fa70d67 100644 --- a/pkg/model/permission.go +++ b/pkg/model/permission.go @@ -8,7 +8,6 @@ type Permission struct { ID uint32 `gorm:"primaryKey" json:"id"` Service string `gorm:"size:100;uniqueIndex:idx_service_permission" json:"service"` PermissionName string `gorm:"size:100;uniqueIndex:idx_service_permission" json:"permission_name"` - Title string `gorm:"size:100" json:"title"` Description string `json:"description"` IsDefault bool `gorm:"default:false" json:"is_default"` CreatedAt time.Time `json:"created_at"` diff --git a/pkg/ratelimit/lua.go b/pkg/ratelimit/lua.go new file mode 100644 index 0000000..7b8b0e7 --- /dev/null +++ b/pkg/ratelimit/lua.go @@ -0,0 +1,92 @@ +package ratelimit + +const ( + // combinedRateLimitScript checks burst, daily and monthly limits in one call + // KEYS[1] - The burst bucket key + // KEYS[2] - The daily quota key + // KEYS[3] - The monthly quota key + // ARGV[1] - Current timestamp in nanoseconds + // ARGV[2] - Cutoff timestamp for old tokens + // ARGV[3] - Maximum burst tokens allowed + // ARGV[4] - Burst expiration time in seconds + // ARGV[5] - Daily maximum requests + // ARGV[6] - Daily expiration time in seconds + // ARGV[7] - Monthly maximum requests + // ARGV[8] - Monthly expiration time in seconds + combinedRateLimitScript = ` +-- Check burst limit (leaky bucket) +local burstKey = KEYS[1] +local now = tonumber(ARGV[1]) +local cutoff = tonumber(ARGV[2]) +local maxBurst = tonumber(ARGV[3]) +local burstExpiration = tonumber(ARGV[4]) + +-- Remove tokens older than the cutoff time +redis.call('ZREMRANGEBYSCORE', burstKey, '0', cutoff) +-- Count current tokens in the bucket +local burstCount = redis.call('ZCARD', burstKey) +local remainingBurst = maxBurst - burstCount +local burstAllowed = remainingBurst > 0 + +-- Check daily quota +local dailyKey = KEYS[2] +local maxDaily = tonumber(ARGV[5]) +local dailyExpiration = tonumber(ARGV[6]) +local dailyCount = redis.call('GET', dailyKey) +local dailyUsed = 0 +if dailyCount then + dailyUsed = tonumber(dailyCount) +end +local remainingDaily = maxDaily - dailyUsed +local dailyAllowed = remainingDaily > 0 + +-- Check monthly quota +local monthlyKey = KEYS[3] +local maxMonthly = tonumber(ARGV[7]) +local monthlyExpiration = tonumber(ARGV[8]) +local monthlyCount = redis.call('GET', monthlyKey) +local monthlyUsed = 0 +if monthlyCount then + monthlyUsed = tonumber(monthlyCount) +end +local remainingMonthly = maxMonthly - monthlyUsed +local monthlyAllowed = remainingMonthly > 0 + +-- Overall allowed if all checks pass +local allowed = burstAllowed and dailyAllowed and monthlyAllowed + +-- If allowed, update all counters +if allowed then + -- Update burst bucket + redis.call('ZADD', burstKey, now, now) + redis.call('EXPIRE', burstKey, burstExpiration) + + -- Update daily quota + if dailyCount then + redis.call('INCR', dailyKey) + else + redis.call('SETEX', dailyKey, dailyExpiration, 1) + end + + -- Update monthly quota + if monthlyCount then + redis.call('INCR', monthlyKey) + else + redis.call('SETEX', monthlyKey, monthlyExpiration, 1) + end + + -- Decrement the remaining counts to account for this request + remainingBurst = remainingBurst - 1 + remainingDaily = remainingDaily - 1 + remainingMonthly = remainingMonthly - 1 +end + +-- Return array with allowed flag and remaining counts +return { + allowed and 1 or 0, + remainingBurst, + remainingDaily, + remainingMonthly +} +` +) diff --git a/pkg/ratelimit/rate_limit.go b/pkg/ratelimit/rate_limit.go new file mode 100644 index 0000000..cdda3ec --- /dev/null +++ b/pkg/ratelimit/rate_limit.go @@ -0,0 +1,124 @@ +package ratelimit + +import ( + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +const ( + // Key prefixes + prefixLeakyBucket = "leaky" + prefixQuota = "quota" + + // Period identifiers + periodDaily = "daily" + periodMonthly = "monthly" + + // Time constants + dailyHours = 24 + monthlyHours = 30 * 24 + bufferTime = 60 // Additional expiration buffer in seconds + + // Script responses + scriptAllowed = 1 +) + +// RateLimitConfig defines all rate limit thresholds +type RateLimitConfig struct { + BurstLimit uint64 // Maximum burst requests + BurstDurationSeconds uint64 // Duration for burst window in seconds + DailyLimit uint64 // Maximum daily requests + MonthlyLimit uint64 // Maximum monthly requests +} + +// RateLimitStatus contains information about remaining limits +type RateLimitStatus struct { + Allowed bool // Whether the current request is allowed + RemainingBurst uint64 // Remaining burst requests + RemainingDaily uint64 // Remaining daily requests + RemainingMonthly uint64 // Remaining monthly requests +} + +// RateLimit provides rate limiting functionality using Redis. +type RateLimit struct { + redis *redis.Client +} + +// New creates a new RateLimit instance. +func New(redisClient *redis.Client) *RateLimit { + return &RateLimit{ + redis: redisClient, + } +} + +// LeakyBucketOptions defines options for the leaky bucket algorithm. +type LeakyBucketOptions struct { + MaxTokens int // Maximum number of tokens in the bucket + Interval time.Duration // Time interval for token replenishment +} + +// QuotaOptions defines options for quota-based rate limiting. +type QuotaOptions struct { + MaxRequests int // Maximum number of requests allowed + Period time.Duration // Time period for the quota +} + +// CheckRateLimits checks all rate limits in a single call and returns remaining counts +// Returns a RateLimitStatus with the allowed flag and remaining counts for all limits +func (rl *RateLimit) CheckRateLimits(ctx context.Context, user uint64, config RateLimitConfig) (*RateLimitStatus, error) { + now := time.Now().UnixNano() + burstKey := formatKey(prefixLeakyBucket, user) + dailyKey := formatKey(fmt.Sprintf("%s:%s", prefixQuota, periodDaily), user) + monthlyKey := formatKey(fmt.Sprintf("%s:%s", prefixQuota, periodMonthly), user) + + // Calculate parameters + burstDuration := time.Duration(config.BurstDurationSeconds) * time.Second + cutoff := now - burstDuration.Nanoseconds() + burstExpiration := int(burstDuration.Seconds()) + bufferTime + dailyExpiration := int(dailyHours * time.Hour.Seconds()) + monthlyExpiration := int(monthlyHours * time.Hour.Seconds()) + + // Execute the Lua script + result, err := rl.redis.Eval(ctx, combinedRateLimitScript, + []string{burstKey, dailyKey, monthlyKey}, + now, // ARGV[1] - Current timestamp + cutoff, // ARGV[2] - Cutoff timestamp + config.BurstLimit, // ARGV[3] - Burst limit + burstExpiration, // ARGV[4] - Burst expiration + config.DailyLimit, // ARGV[5] - Daily limit + dailyExpiration, // ARGV[6] - Daily expiration + config.MonthlyLimit, // ARGV[7] - Monthly limit + monthlyExpiration, // ARGV[8] - Monthly expiration + ).Result() + + if err != nil { + return nil, fmt.Errorf("failed to execute rate limit script: %w", err) + } + + // Parse the response + values, ok := result.([]interface{}) + if !ok || len(values) != 4 { + return nil, fmt.Errorf("unexpected response format from rate limit script") + } + + // Convert values to uint64 + allowed := values[0].(int64) == scriptAllowed + remainingBurst, _ := values[1].(int64) + remainingDaily, _ := values[2].(int64) + remainingMonthly, _ := values[3].(int64) + + return &RateLimitStatus{ + Allowed: allowed, + RemainingBurst: uint64(remainingBurst), + RemainingDaily: uint64(remainingDaily), + RemainingMonthly: uint64(remainingMonthly), + }, nil +} + +// formatKey creates a consistent format for Redis keys +func formatKey(prefix string, key uint64) string { + return fmt.Sprintf("%s:%d", prefix, key) +} diff --git a/pkg/rpc/dev.go b/pkg/rpc/dev.go new file mode 100644 index 0000000..e595ddb --- /dev/null +++ b/pkg/rpc/dev.go @@ -0,0 +1,195 @@ +package rpc + +import ( + "context" + "fmt" + "net/http" + "time" + + "git.itzana.me/strafesnet/dev-portal/pkg/cache" + "git.itzana.me/strafesnet/dev-portal/pkg/datastore" + "git.itzana.me/strafesnet/dev-portal/pkg/model" + "git.itzana.me/strafesnet/dev-portal/pkg/ratelimit" + "git.itzana.me/strafesnet/go-grpc/dev" +) + +// Error messages +const ( + ErrEmptyAPIKey = "empty API key" + ErrEmptyService = "empty service name" + ErrEmptyPermission = "empty permission name" + ErrAppInactive = "application is inactive" + ErrRateLimitExceeded = "rate limit exceeded" + ErrUnauthorized = "unauthorized request" +) + +// Cache settings +const ( + ApplicationCacheDuration = 5 * time.Minute +) + +// Dev implements the DevServiceServer interface +type Dev struct { + *dev.UnimplementedDevServiceServer + Store datastore.Datastore + Limiter *ratelimit.RateLimit + Cache *cache.Cache +} + +// Validate checks if an API key is valid and has the required permissions +func (d Dev) Validate(ctx context.Context, request *dev.APIValidationRequest) (*dev.APIValidationResponse, error) { + // Check for context cancellation + if ctx.Err() != nil { + return errorResponse(fmt.Errorf("context error: %w", ctx.Err()), http.StatusInternalServerError) + } + + // Validate request parameters + if err := d.validateRequestParams(request); err != nil { + return errorResponse(err, http.StatusBadRequest) + } + + // Get application data + appCache, err := d.getApplicationData(ctx, request.Key) + if err != nil { + return errorResponse(err, http.StatusInternalServerError) + } + + // Check if application is active + if !appCache.Active { + return buildResponse(appCache, &ratelimit.RateLimitStatus{}, false, ErrAppInactive, http.StatusForbidden), nil + } + + // Check rate limits + limit, err := d.checkRateLimits(ctx, appCache) + if err != nil { + return buildResponse(appCache, createDefaultRateLimitStatus(), false, + fmt.Sprintf("failed to check rate limits: %v", err), http.StatusInternalServerError), nil + } + + // Rate limit exceeded + if !limit.Allowed { + return buildResponse(appCache, limit, false, ErrRateLimitExceeded, http.StatusTooManyRequests), nil + } + + // Check permission + if hasRequiredPermission(appCache.Permissions, request.Service, request.Permission) { + return buildResponse(appCache, limit, true, "", 0), nil + } + + return buildResponse(appCache, limit, false, ErrUnauthorized, http.StatusUnauthorized), nil +} + +// validateRequestParams validates the basic parameters of the request +func (d Dev) validateRequestParams(request *dev.APIValidationRequest) error { + if request.Key == "" { + return fmt.Errorf(ErrEmptyAPIKey) + } + if request.Service == "" { + return fmt.Errorf(ErrEmptyService) + } + if request.Permission == "" { + return fmt.Errorf(ErrEmptyPermission) + } + return nil +} + +// checkRateLimits checks if the application has exceeded its rate limits +func (d Dev) checkRateLimits(ctx context.Context, appCache *model.ApplicationCache) (*ratelimit.RateLimitStatus, error) { + rateLimitConfig := ratelimit.RateLimitConfig{ + BurstLimit: appCache.RateLimit.BurstLimit, + BurstDurationSeconds: appCache.RateLimit.BurstDuration, + DailyLimit: appCache.RateLimit.DailyLimit, + MonthlyLimit: appCache.RateLimit.MonthlyLimit, + } + + return d.Limiter.CheckRateLimits(ctx, appCache.UserID, rateLimitConfig) +} + +// createDefaultRateLimitStatus creates a default rate limit status with all limits at zero +func createDefaultRateLimitStatus() *ratelimit.RateLimitStatus { + return &ratelimit.RateLimitStatus{ + Allowed: false, + RemainingBurst: 0, + RemainingDaily: 0, + RemainingMonthly: 0, + } +} + +// getApplicationData retrieves application data from cache or datastore +func (d Dev) getApplicationData(ctx context.Context, apiKey string) (*model.ApplicationCache, error) { + // Check for context cancellation + if ctx.Err() != nil { + return nil, fmt.Errorf("context error: %w", ctx.Err()) + } + + // Try to get from cache first + appCache, err := d.Cache.GetApplicationByAPIKey(ctx, apiKey) + if err == nil { + return appCache, nil + } + + // Not in cache, retrieve from store + app, err := d.Store.GetApplicationByAPIKey(ctx, apiKey) + if err != nil { + return nil, fmt.Errorf("failed to retrieve application: %w", err) + } + + user, err := d.Store.GetUser(ctx, app.UserID) + if err != nil { + return nil, fmt.Errorf("failed to retrieve user: %w", err) + } + + appCache = &model.ApplicationCache{ + Name: app.Name, + UserID: app.UserID, + APIKey: app.APIKey, + Permissions: app.Permissions, + Active: app.Active, + RateLimit: user.RateLimit, + } + + // Cache the application data - log the error but don't fail the operation + if err := d.Cache.SetApplicationByAPIKey(ctx, appCache, ApplicationCacheDuration); err != nil { + // Log the error but continue since we have the data + fmt.Printf("warning: failed to cache application: %v\n", err) + } + + return appCache, nil +} + +// errorResponse creates a standardized error response +func errorResponse(err error, statusCode int32) (*dev.APIValidationResponse, error) { + return &dev.APIValidationResponse{ + Valid: false, + ErrorMessage: err.Error(), + StatusCode: statusCode, + }, nil +} + +// hasRequiredPermission checks if the permissions list contains the required service and permission +func hasRequiredPermission(permissions []model.Permission, service, permissionName string) bool { + for _, p := range permissions { + if p.PermissionName == permissionName && p.Service == service { + return true + } + } + return false +} + +// buildResponse constructs a validation response with common fields +func buildResponse(app *model.ApplicationCache, limit *ratelimit.RateLimitStatus, valid bool, errorMsg string, statusCode int32) *dev.APIValidationResponse { + return &dev.APIValidationResponse{ + Valid: valid, + ErrorMessage: errorMsg, + StatusCode: statusCode, + RemainingBurst: limit.RemainingBurst, + RemainingDaily: limit.RemainingDaily, + RemainingMonthly: limit.RemainingMonthly, + BurstLimit: app.RateLimit.BurstLimit, + BurstDurationSeconds: app.RateLimit.BurstDuration, + DailyLimit: app.RateLimit.DailyLimit, + MonthlyLimit: app.RateLimit.MonthlyLimit, + UserID: app.UserID, + Application: app.Name, + } +}