Skip to content

secret storage #3128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 51 commits into from
Jul 13, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
594866e
feat(storage): add storage table
fiftin Jul 2, 2025
fb69eb7
feat(db): add secret storage
fiftin Jul 2, 2025
d97dbb8
feat(stoarge): add service
fiftin Jul 2, 2025
68798dd
Merge branch 'develop' into secret_storage
fiftin Jul 2, 2025
35144ad
feat(keys): add storage model
fiftin Jul 4, 2025
7b0413a
feat(keys): model fields
fiftin Jul 4, 2025
ae4e375
feat(keys): add storages page
fiftin Jul 4, 2025
ea4e7bc
feat(keys): add secret storages controller
fiftin Jul 4, 2025
9f68053
refactor: use services for access key descryption
fiftin Jul 5, 2025
db50fff
feat(keys): add vault support
fiftin Jul 5, 2025
fffa2a0
feat(keys): add endpoints
fiftin Jul 6, 2025
0dfea52
fix(keys): fill installer interface
fiftin Jul 6, 2025
c44c9fd
fix: null pointers of services
fiftin Jul 6, 2025
48987c1
fix(api): incorrect usage middleware
fiftin Jul 6, 2025
231d7cc
fix(secrets): fix sql query
fiftin Jul 7, 2025
1fcf080
feat(secrets): manipulation of storage
fiftin Jul 7, 2025
b443b77
feat(storage): works
fiftin Jul 7, 2025
7c34db1
fix(storage): do not clear access key
fiftin Jul 8, 2025
f1c9983
feat(storage): vault settings
fiftin Jul 8, 2025
535ca0a
feat(storage): choose storage for access key
fiftin Jul 8, 2025
390941e
refactor(secrets): copy method to serializer
fiftin Jul 8, 2025
2f9858a
feat(secrets): add access key service
fiftin Jul 8, 2025
1fad415
fix: merge conflict
fiftin Jul 8, 2025
5017b57
test: fix tests
fiftin Jul 8, 2025
6019d0a
test: fix tests
fiftin Jul 8, 2025
6343f1f
fix: services
fiftin Jul 9, 2025
b6eec50
feat(secrets): store keys to vault
fiftin Jul 9, 2025
68c8cfa
feat(secrets): delete storage endpoint
fiftin Jul 9, 2025
a824a87
feat(secrets): delete storage endpoint
fiftin Jul 9, 2025
8ecfc7d
fix(secrets): correct condition
fiftin Jul 9, 2025
2d01ae0
feat: add icon
fiftin Jul 10, 2025
4b17a9b
feat(secrets): add menu to new storage button
fiftin Jul 10, 2025
73c9e26
feat(secrets): add icon for strage
fiftin Jul 10, 2025
d1d0849
fix(secrets): allow empty token field for existing storage
fiftin Jul 11, 2025
99b157a
feat(secrets): readonly secret storage
fiftin Jul 11, 2025
7079988
feat(secrets): add readonly badge
fiftin Jul 11, 2025
9fd537b
feat(secrets): delete secret from vault
fiftin Jul 11, 2025
f7fec52
feat(secrets): delete remote secret when delete access key
fiftin Jul 11, 2025
0fa8972
feat(secrets): use access key service
fiftin Jul 11, 2025
76b0e99
security(secrets): upgrade dep
fiftin Jul 11, 2025
3bb80cc
refactor(secrets): split to foss and pro
fiftin Jul 12, 2025
12592a5
refactor(service): rename package name
fiftin Jul 12, 2025
ce880ae
refactor(features): add GetFeatures function
fiftin Jul 12, 2025
ddb62d0
feat(secrets): update ui
fiftin Jul 12, 2025
b9b84a3
fix: merge conflict
fiftin Jul 12, 2025
9261149
ci: dir to /tmp for deps
fiftin Jul 12, 2025
83f7806
refactor(pro): add mock pro dir
fiftin Jul 13, 2025
06f44da
chore: ingore go.work
fiftin Jul 13, 2025
85a2bb3
test(secrets): fix mock
fiftin Jul 13, 2025
1e06c7a
fix(secrets): sqlite migration
fiftin Jul 13, 2025
e24ffde
ci: fix docker image build
fiftin Jul 13, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,7 @@ __debug_bin*

/events.log
/tasks.log
/task_results.log
/task_results.log
/pro_impl/
/go.work
/go.work.sum
1 change: 1 addition & 0 deletions Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ tasks:
# - go install github.com/go-swagger/go-swagger/cmd/swagger@{{ .SWAGGER_VERSION }}
- go install github.com/goreleaser/goreleaser@{{ .GORELEASER_VERSION }}
# - go install github.com/golangci/golangci-lint/cmd/golangci-lint@{{ .GOLINTER_VERSION }}
dir: /tmp

deps:be:
desc: Vendor application dependencies
Expand Down
11 changes: 10 additions & 1 deletion api/api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,16 @@ func TestApiPing(t *testing.T) {
req, _ := http.NewRequest("GET", "/api/ping", nil)
rr := httptest.NewRecorder()

r := Route(nil, nil)
r := Route(
nil,
nil,
nil,
nil,
nil,
nil,
nil,
nil,
)

r.ServeHTTP(rr, req)

Expand Down
54 changes: 2 additions & 52 deletions api/apps.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,63 +6,13 @@ import (
"fmt"
"github.com/semaphoreui/semaphore/api/helpers"
"github.com/semaphoreui/semaphore/db"
"github.com/semaphoreui/semaphore/pkg/conv"
"github.com/semaphoreui/semaphore/util"
"net/http"
"reflect"
"sort"
"strings"
)

func structToFlatMap(obj any) map[string]any {
result := make(map[string]any)
val := reflect.ValueOf(obj)
typ := reflect.TypeOf(obj)

if typ.Kind() == reflect.Ptr {
val = val.Elem()
typ = typ.Elem()
}

if typ.Kind() != reflect.Struct {
return result
}

// Iterate over the struct fields
for i := 0; i < val.NumField(); i++ {
field := val.Field(i)
fieldType := typ.Field(i)
jsonTag := fieldType.Tag.Get("json")

// Use the json tag if it is set, otherwise use the field name
fieldName := jsonTag
if fieldName == "" || fieldName == "-" {
fieldName = fieldType.Name
} else {
// Handle the case where the json tag might have options like `json:"name,omitempty"`
fieldName = strings.Split(fieldName, ",")[0]
}

// Check if the field is a struct itself
if field.Kind() == reflect.Struct {
// Convert nested struct to map
nestedMap := structToFlatMap(field.Interface())
// Add nested map to result with a prefixed key
for k, v := range nestedMap {
result[fieldName+"."+k] = v
}
} else if (field.Kind() == reflect.Ptr ||
field.Kind() == reflect.Array ||
field.Kind() == reflect.Slice ||
field.Kind() == reflect.Map) && field.IsNil() {
result[fieldName] = nil
} else {
result[fieldName] = field.Interface()
}
}

return result
}

func validateAppID(str string) error {
return nil
}
Expand Down Expand Up @@ -170,7 +120,7 @@ func setApp(w http.ResponseWriter, r *http.Request) {
return
}

options := structToFlatMap(app)
options := conv.StructToFlatMap(app)

for k, v := range options {
t := reflect.TypeOf(v)
Expand Down
3 changes: 2 additions & 1 deletion api/apps_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package api

import (
"fmt"
"github.com/semaphoreui/semaphore/pkg/conv"
"testing"
)

Expand Down Expand Up @@ -32,7 +33,7 @@ func TestStructToMap(t *testing.T) {
}

// Convert the struct to a flat map
flatMap := structToFlatMap(&p)
flatMap := conv.StructToFlatMap(&p)

if flatMap["address.city"] != "New York" {
t.Fail()
Expand Down
17 changes: 14 additions & 3 deletions api/integration.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"encoding/json"
"fmt"
"github.com/semaphoreui/semaphore/pkg/conv"
"github.com/semaphoreui/semaphore/services/server"
task2 "github.com/semaphoreui/semaphore/services/tasks"
"io"
"net/http"
Expand Down Expand Up @@ -44,7 +45,17 @@ func hmacHashPayload(secret string, payloadBody []byte) string {
return fmt.Sprintf("%x", sum)
}

func ReceiveIntegration(w http.ResponseWriter, r *http.Request) {
type IntegrationController struct {
integrationService server.IntegrationService
}

func NewIntegrationController(integrationService server.IntegrationService) *IntegrationController {
return &IntegrationController{
integrationService: integrationService,
}
}

func (c *IntegrationController) ReceiveIntegration(w http.ResponseWriter, r *http.Request) {

var err error

Expand Down Expand Up @@ -95,7 +106,7 @@ func ReceiveIntegration(w http.ResponseWriter, r *http.Request) {
panic("")
}

err = db.FillIntegration(store, &integration)
err = c.integrationService.FillIntegration(&integration)
if err != nil {
log.Error(err)
return
Expand Down Expand Up @@ -288,7 +299,7 @@ func RunIntegration(integration db.Integration, project db.Project, r *http.Requ
log.Error(err)
return
}

pool := helpers.GetFromContext(r, "task_pool").(*task2.TaskPool)

_, err = pool.AddTask(taskDefinition, nil, "", integration.ProjectID, tpl.App.NeedTaskAlias())
Expand Down
47 changes: 33 additions & 14 deletions api/projects/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,29 @@ import (
"fmt"
"github.com/semaphoreui/semaphore/api/helpers"
"github.com/semaphoreui/semaphore/db"
"github.com/semaphoreui/semaphore/services/server"
"net/http"
)

func updateEnvironmentSecrets(store db.Store, env db.Environment) error {
type EnvironmentController struct {
accessKeyRepo db.AccessKeyManager
accessKeyService server.AccessKeyService
encryptionService server.AccessKeyEncryptionService
}

func NewEnvironmentController(
accessKeyRepo db.AccessKeyManager,
encryptionService server.AccessKeyEncryptionService,
accessKeyService server.AccessKeyService,
) *EnvironmentController {
return &EnvironmentController{
accessKeyRepo: accessKeyRepo,
accessKeyService: accessKeyService,
encryptionService: encryptionService,
}
}

func (c *EnvironmentController) updateEnvironmentSecrets(env db.Environment) error {
for _, secret := range env.Secrets {
err := secret.Validate()
if err != nil {
Expand All @@ -18,7 +37,7 @@ func updateEnvironmentSecrets(store db.Store, env db.Environment) error {

switch secret.Operation {
case db.EnvironmentSecretCreate:
key, err = store.CreateAccessKey(db.AccessKey{
key, err = c.accessKeyService.CreateAccessKey(db.AccessKey{
Name: secret.Name,
String: secret.Secret,
EnvironmentID: &env.ID,
Expand All @@ -27,7 +46,7 @@ func updateEnvironmentSecrets(store db.Store, env db.Environment) error {
Owner: secret.Type.GetAccessKeyOwner(),
})
case db.EnvironmentSecretDelete:
key, err = store.GetAccessKey(env.ProjectID, secret.ID)
key, err = c.accessKeyRepo.GetAccessKey(env.ProjectID, secret.ID)

if err != nil {
continue
Expand All @@ -37,9 +56,9 @@ func updateEnvironmentSecrets(store db.Store, env db.Environment) error {
continue
}

err = store.DeleteAccessKey(env.ProjectID, secret.ID)
err = c.accessKeyService.DeleteAccessKey(env.ProjectID, secret.ID)
case db.EnvironmentSecretUpdate:
key, err = store.GetAccessKey(env.ProjectID, secret.ID)
key, err = c.accessKeyRepo.GetAccessKey(env.ProjectID, secret.ID)

if err != nil {
continue
Expand All @@ -61,15 +80,15 @@ func updateEnvironmentSecrets(store db.Store, env db.Environment) error {
updateKey.OverrideSecret = true
}

err = store.UpdateAccessKey(updateKey)
err = c.accessKeyService.UpdateAccessKey(updateKey)
}
}

return nil
}

// EnvironmentMiddleware ensures an environment exists and loads it to the context
func EnvironmentMiddleware(next http.Handler) http.Handler {
func (c *EnvironmentController) EnvironmentMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
project := helpers.GetFromContext(r, "project").(db.Project)
envID, err := helpers.GetIntParam("environment_id", w, r)
Expand All @@ -85,7 +104,7 @@ func EnvironmentMiddleware(next http.Handler) http.Handler {
return
}

if err = db.FillEnvironmentSecrets(helpers.Store(r), &env, false); err != nil {
if err = c.encryptionService.FillEnvironmentSecrets(&env, false); err != nil {
helpers.WriteError(w, err)
return
}
Expand Down Expand Up @@ -128,7 +147,7 @@ func GetEnvironment(w http.ResponseWriter, r *http.Request) {
}

// UpdateEnvironment updates an existing environment in the database
func UpdateEnvironment(w http.ResponseWriter, r *http.Request) {
func (c *EnvironmentController) UpdateEnvironment(w http.ResponseWriter, r *http.Request) {
oldEnv := helpers.GetFromContext(r, "environment").(db.Environment)
var env db.Environment
if !helpers.Bind(w, r, &env) {
Expand Down Expand Up @@ -162,7 +181,7 @@ func UpdateEnvironment(w http.ResponseWriter, r *http.Request) {
Description: fmt.Sprintf("Environment %s updated", env.Name),
})

if err := updateEnvironmentSecrets(helpers.Store(r), env); err != nil {
if err := c.updateEnvironmentSecrets(env); err != nil {
helpers.WriteError(w, err)
return
}
Expand All @@ -171,7 +190,7 @@ func UpdateEnvironment(w http.ResponseWriter, r *http.Request) {
}

// AddEnvironment creates an environment in the database
func AddEnvironment(w http.ResponseWriter, r *http.Request) {
func (c *EnvironmentController) AddEnvironment(w http.ResponseWriter, r *http.Request) {
project := helpers.GetFromContext(r, "project").(db.Project)
var env db.Environment

Expand Down Expand Up @@ -199,9 +218,9 @@ func AddEnvironment(w http.ResponseWriter, r *http.Request) {
Description: fmt.Sprintf("Environment %s created", newEnv.Name),
})

if err = updateEnvironmentSecrets(helpers.Store(r), newEnv); err != nil {
//helpers.WriteError(w, err)
//return
if err = c.updateEnvironmentSecrets(newEnv); err != nil {
helpers.WriteError(w, err)
return
}

// Reload env
Expand Down
28 changes: 21 additions & 7 deletions api/projects/keys.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,27 @@
package projects

import (
"errors"
"fmt"
"github.com/semaphoreui/semaphore/services/server"
"net/http"

"github.com/semaphoreui/semaphore/api/helpers"
"github.com/semaphoreui/semaphore/db"
)

type KeyController struct {
accessKeyService server.AccessKeyService
}

func NewKeyController(
accessKeyService server.AccessKeyService,
) *KeyController {
return &KeyController{
accessKeyService: accessKeyService,
}
}

// KeyMiddleware ensures a key exists and loads it to the context
func KeyMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
Expand Down Expand Up @@ -62,7 +76,7 @@ func GetKeys(w http.ResponseWriter, r *http.Request) {
}

// AddKey adds a new key to the database
func AddKey(w http.ResponseWriter, r *http.Request) {
func (c *KeyController) AddKey(w http.ResponseWriter, r *http.Request) {
project := helpers.GetFromContext(r, "project").(db.Project)
var key db.AccessKey

Expand All @@ -84,7 +98,7 @@ func AddKey(w http.ResponseWriter, r *http.Request) {
return
}

newKey, err := helpers.Store(r).CreateAccessKey(key)
newKey, err := c.accessKeyService.CreateAccessKey(key)

if err != nil {
helpers.WriteError(w, err)
Expand All @@ -111,7 +125,7 @@ func AddKey(w http.ResponseWriter, r *http.Request) {

// UpdateKey updates key in database
// nolint: gocyclo
func UpdateKey(w http.ResponseWriter, r *http.Request) {
func (c *KeyController) UpdateKey(w http.ResponseWriter, r *http.Request) {
var key db.AccessKey
oldKey := helpers.GetFromContext(r, "accessKey").(db.AccessKey)

Expand All @@ -136,7 +150,7 @@ func UpdateKey(w http.ResponseWriter, r *http.Request) {
}
}

err = helpers.Store(r).UpdateAccessKey(key)
err = c.accessKeyService.UpdateAccessKey(key)
if err != nil {
helpers.WriteError(w, err)
return
Expand All @@ -154,11 +168,11 @@ func UpdateKey(w http.ResponseWriter, r *http.Request) {
}

// RemoveKey deletes a key from the database
func RemoveKey(w http.ResponseWriter, r *http.Request) {
func (c *KeyController) RemoveKey(w http.ResponseWriter, r *http.Request) {
key := helpers.GetFromContext(r, "accessKey").(db.AccessKey)

err := helpers.Store(r).DeleteAccessKey(*key.ProjectID, key.ID)
if err == db.ErrInvalidOperation {
err := c.accessKeyService.DeleteAccessKey(*key.ProjectID, key.ID)
if errors.Is(err, db.ErrInvalidOperation) {
helpers.WriteJSON(w, http.StatusBadRequest, map[string]any{
"error": "Access Key is in use by one or more templates",
"inUse": true,
Expand Down
4 changes: 2 additions & 2 deletions api/projects/project.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"github.com/gorilla/mux"
"github.com/semaphoreui/semaphore/api/helpers"
"github.com/semaphoreui/semaphore/db"
"github.com/semaphoreui/semaphore/services"
"github.com/semaphoreui/semaphore/services/server"
"github.com/semaphoreui/semaphore/util"
log "github.com/sirupsen/logrus"
"net/http"
Expand Down Expand Up @@ -63,7 +63,7 @@ func GetMustCanMiddleware(permissions db.ProjectUserPermission) mux.MiddlewareFu
}

type ProjectController struct {
ProjectService services.ProjectService
ProjectService server.ProjectService
}

func (c *ProjectController) UpdateProject(w http.ResponseWriter, r *http.Request) {
Expand Down
Loading
Loading