Validate and import agent skills into versioned server storage
Adds a SKILL.md bundle validator, immutable per-skill versions with content digests and deduplication, protocol requests to import, list and show skills, and the pipelines skills import/list/show commands. Imports are serialised so concurrent imports never share a version number.
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
# Skills
|
||||
|
||||
`pipelines skills import FOLDER` validates a skill locally and, after confirmation,
|
||||
sends its bytes to the server. A bundle contains one root `SKILL.md` with YAML
|
||||
frontmatter (`name` and `description`) and may contain only `references/`,
|
||||
`scripts/`, and `assets/`. Symlinks, traversal paths, nested `SKILL.md` files,
|
||||
executables outside `scripts/`, and oversized bundles are rejected. Files are
|
||||
fingerprinted in sorted path order with SHA-256; importing the same digest is
|
||||
idempotent. The server keeps immutable versions in SQLite metadata and stores
|
||||
their files below its data directory. Skills are metadata-only at this stage:
|
||||
they are not executed or injected into stages or chats.
|
||||
|
||||
Use `pipelines skills list` and `pipelines skills show NAME [--version N]` to
|
||||
inspect imported versions.
|
||||
@@ -285,6 +285,9 @@ func parse(f *flag.FlagSet, args []string) error {
|
||||
}
|
||||
|
||||
func (a App) Run(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "skills" {
|
||||
return a.skillsCommand(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "choose-model" {
|
||||
return a.chooseModel(ctx, args[1:])
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
// Help is also the public command inventory for agents and parity checks.
|
||||
var usage = map[string]string{
|
||||
"skills": "skills import <folder> [--yes] | skills list | skills show <name> [--version N]",
|
||||
"repair-session-export": "repair-session-export RUN_ID SEQUENCE [--json]",
|
||||
"provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]",
|
||||
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"git.a-wan.net/alan/pipelines/internal/client"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/skills"
|
||||
)
|
||||
|
||||
func (a App) skillsCommand(ctx context.Context, args []string) error {
|
||||
if len(args) < 1 {
|
||||
return fmt.Errorf("usage: pipelines skills import|list|show")
|
||||
}
|
||||
if args[0] == "help" || args[0] == "--help" || args[0] == "-h" {
|
||||
_, err := fmt.Fprintln(a.Out, "Usage: pipelines skills import <folder> [--yes] | skills list | skills show <name> [--version N]")
|
||||
return err
|
||||
}
|
||||
// Accept flags after the folder, matching the rest of the Pipelines CLI.
|
||||
var flags, positionals []string
|
||||
for i := 1; i < len(args); i++ {
|
||||
if args[i] == "--yes" || args[i] == "--json" || args[i] == "-json" {
|
||||
flags = append(flags, args[i])
|
||||
continue
|
||||
}
|
||||
if args[i] == "--url" || args[i] == "--version" {
|
||||
if i+1 >= len(args) {
|
||||
return fmt.Errorf("%s needs a value", args[i])
|
||||
}
|
||||
flags = append(flags, args[i], args[i+1])
|
||||
i++
|
||||
continue
|
||||
}
|
||||
positionals = append(positionals, args[i])
|
||||
}
|
||||
args = append([]string{args[0]}, append(flags, positionals...)...)
|
||||
sub := args[0]
|
||||
fs := flag.NewFlagSet("skills "+sub, flag.ContinueOnError)
|
||||
fs.SetOutput(a.Err)
|
||||
url := envOr("PIPELINES_URL", "ws://127.0.0.1:8080/connect")
|
||||
yes := false
|
||||
version := 0
|
||||
jsonMode := false
|
||||
fs.StringVar(&url, "url", url, "Pipeline server WebSocket URL")
|
||||
fs.BoolVar(&yes, "yes", false, "Confirm import without prompting")
|
||||
fs.BoolVar(&jsonMode, "json", false, "Machine-readable JSON")
|
||||
fs.IntVar(&version, "version", 0, "Immutable version (0 = latest)")
|
||||
if err := fs.Parse(args[1:]); err != nil {
|
||||
return err
|
||||
}
|
||||
pos := fs.Args()
|
||||
if sub == "import" {
|
||||
if len(pos) != 1 {
|
||||
return fmt.Errorf("skills import requires a folder")
|
||||
}
|
||||
b, e := skills.Validate(pos[0])
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if !yes {
|
||||
if e = a.confirmAction(fmt.Sprintf("Import skill %s (%d files, %d bytes, digest %s)?", b.Name, len(b.Files), b.TotalBytes, b.Digest), false); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
c, e := client.Open(ctx, url)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
defer c.Close()
|
||||
files := make([]protocol.SkillFile, len(b.Files))
|
||||
for i, f := range b.Files {
|
||||
files[i] = protocol.SkillFile{Path: f.Path, Bytes: f.Bytes}
|
||||
}
|
||||
r, e := c.Call(ctx, protocol.Message{Type: protocol.ImportSkill, Name: b.Name, Description: b.Description, Digest: b.Digest, SourcePath: pos[0], SkillFiles: files})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if r.SkillVersion == nil {
|
||||
return fmt.Errorf("server returned no skill version")
|
||||
}
|
||||
return a.outputSkill(r.SkillVersion, jsonMode)
|
||||
}
|
||||
if sub != "list" && sub != "show" {
|
||||
return fmt.Errorf("unknown skills command %q", sub)
|
||||
}
|
||||
c, e := client.Open(ctx, url)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
defer c.Close()
|
||||
if sub == "list" {
|
||||
if len(pos) != 0 {
|
||||
return fmt.Errorf("skills list takes no arguments")
|
||||
}
|
||||
r, e := c.Call(ctx, protocol.Message{Type: protocol.ListSkills})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if jsonMode {
|
||||
return a.output(r)
|
||||
}
|
||||
for _, s := range r.SkillSummaries {
|
||||
fmt.Fprintf(a.Out, "%s\tv%d\t%s\n", s.Name, s.LatestVersion, s.Digest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(pos) != 1 {
|
||||
return fmt.Errorf("skills show requires a name")
|
||||
}
|
||||
r, e := c.Call(ctx, protocol.Message{Type: protocol.ShowSkill, Name: pos[0], Version: version})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if r.SkillVersion == nil {
|
||||
return fmt.Errorf("server returned no skill")
|
||||
}
|
||||
return a.outputSkill(r.SkillVersion, jsonMode)
|
||||
}
|
||||
func (a App) outputSkill(s *protocol.SkillVersion, jsonMode bool) error {
|
||||
if jsonMode {
|
||||
return a.output(s)
|
||||
}
|
||||
fmt.Fprintf(a.Out, "%s v%d\t%s\n%s\n", s.Name, s.Version, s.Digest, s.SkillMD)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/skills"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (s *Store) initSkills() error {
|
||||
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS skills (name TEXT PRIMARY KEY, description TEXT NOT NULL, latest_version INTEGER NOT NULL, digest TEXT NOT NULL, imported_at TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS skill_versions (name TEXT NOT NULL REFERENCES skills(name), version INTEGER NOT NULL, digest TEXT NOT NULL, source_path TEXT NOT NULL, imported_at TEXT NOT NULL, PRIMARY KEY(name,version), UNIQUE(name,digest));
|
||||
CREATE TABLE IF NOT EXISTS skill_files (name TEXT NOT NULL, version INTEGER NOT NULL, path TEXT NOT NULL, size INTEGER NOT NULL, PRIMARY KEY(name,version,path), FOREIGN KEY(name,version) REFERENCES skill_versions(name,version));`)
|
||||
return err
|
||||
}
|
||||
|
||||
// skillImports serialises imports in this server process, so the
|
||||
// deduplication check and the next-version allocation below see every
|
||||
// earlier import's committed version and never hand out the same number.
|
||||
var skillImports sync.Mutex
|
||||
|
||||
func (s *Store) ImportSkill(ctx context.Context, name, description, digest, source string, files []protocol.SkillFile) (protocol.SkillVersion, error) {
|
||||
skillImports.Lock()
|
||||
defer skillImports.Unlock()
|
||||
if err := skills.ValidateName(name); err != nil {
|
||||
return protocol.SkillVersion{}, err
|
||||
}
|
||||
if digest == "" || len(files) == 0 {
|
||||
return protocol.SkillVersion{}, fmt.Errorf("invalid skill bundle")
|
||||
}
|
||||
if len(files) > skills.MaxFiles {
|
||||
return protocol.SkillVersion{}, fmt.Errorf("skill has more than %d files", skills.MaxFiles)
|
||||
}
|
||||
ordered := append([]protocol.SkillFile(nil), files...)
|
||||
sort.Slice(ordered, func(i, j int) bool { return ordered[i].Path < ordered[j].Path })
|
||||
h := sha256.New()
|
||||
total := 0
|
||||
for _, f := range ordered {
|
||||
if f.Path == "" || strings.HasPrefix(f.Path, "/") || filepath.Clean(f.Path) != f.Path || strings.HasPrefix(filepath.ToSlash(f.Path), "../") || strings.Contains(f.Path, "\\") {
|
||||
return protocol.SkillVersion{}, fmt.Errorf("invalid skill file path: %s", f.Path)
|
||||
}
|
||||
if len(f.Bytes) > skills.MaxFileBytes {
|
||||
return protocol.SkillVersion{}, fmt.Errorf("file %s exceeds %d bytes", f.Path, skills.MaxFileBytes)
|
||||
}
|
||||
total += len(f.Bytes)
|
||||
fmt.Fprintf(h, "%s\x00%d\x00", f.Path, len(f.Bytes))
|
||||
h.Write(f.Bytes)
|
||||
}
|
||||
if total > skills.MaxTotalBytes || hex.EncodeToString(h.Sum(nil)) != digest {
|
||||
return protocol.SkillVersion{}, fmt.Errorf("skill digest or size limit is invalid")
|
||||
}
|
||||
var existing protocol.SkillVersion
|
||||
err := s.db.QueryRowContext(ctx, `SELECT version, source_path, imported_at FROM skill_versions WHERE name=? AND digest=?`, name, digest).Scan(&existing.Version, &existing.SourcePath, &existing.ImportedAt)
|
||||
if err == nil {
|
||||
existing.SkillSummary = protocol.SkillSummary{Name: name, Description: description, LatestVersion: existing.Version, Digest: digest, ImportedAt: existing.ImportedAt}
|
||||
existing.SkillMD, _ = s.skillMD(name, existing.Version)
|
||||
existing.Manifest, _ = s.skillManifest(ctx, name, existing.Version)
|
||||
return existing, nil
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return existing, err
|
||||
}
|
||||
var next int
|
||||
_ = s.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(version),0)+1 FROM skill_versions WHERE name=?`, name).Scan(&next)
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return existing, err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO skills(name,description,latest_version,digest,imported_at) VALUES(?,?,?,?,?) ON CONFLICT(name) DO UPDATE SET description=excluded.description,latest_version=excluded.latest_version,digest=excluded.digest,imported_at=excluded.imported_at`, name, description, next, digest, now); err != nil {
|
||||
tx.Rollback()
|
||||
return existing, err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO skill_versions(name,version,digest,source_path,imported_at) VALUES(?,?,?,?,?)`, name, next, digest, source, now); err != nil {
|
||||
tx.Rollback()
|
||||
return existing, err
|
||||
}
|
||||
dir := filepath.Join(s.dir, "skills", name, fmt.Sprintf("v%d", next))
|
||||
if err = os.MkdirAll(dir, 0700); err != nil {
|
||||
tx.Rollback()
|
||||
return existing, err
|
||||
}
|
||||
for _, f := range files {
|
||||
p := filepath.Join(dir, filepath.FromSlash(f.Path))
|
||||
if err = os.MkdirAll(filepath.Dir(p), 0700); err != nil {
|
||||
tx.Rollback()
|
||||
return existing, err
|
||||
}
|
||||
if err = os.WriteFile(p, f.Bytes, 0600); err != nil {
|
||||
tx.Rollback()
|
||||
return existing, err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO skill_files(name,version,path,size) VALUES(?,?,?,?)`, name, next, f.Path, len(f.Bytes)); err != nil {
|
||||
tx.Rollback()
|
||||
return existing, err
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return existing, err
|
||||
}
|
||||
return s.GetSkill(ctx, name, next)
|
||||
}
|
||||
func (s *Store) ListSkills(ctx context.Context) ([]protocol.SkillSummary, error) {
|
||||
rows, e := s.db.QueryContext(ctx, `SELECT name,description,latest_version,digest,imported_at FROM skills ORDER BY name`)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []protocol.SkillSummary
|
||||
for rows.Next() {
|
||||
var x protocol.SkillSummary
|
||||
if e = rows.Scan(&x.Name, &x.Description, &x.LatestVersion, &x.Digest, &x.ImportedAt); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
out = append(out, x)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
func (s *Store) GetSkill(ctx context.Context, name string, version int) (protocol.SkillVersion, error) {
|
||||
var x protocol.SkillVersion
|
||||
var desc string
|
||||
if version == 0 {
|
||||
if e := s.db.QueryRowContext(ctx, `SELECT description,latest_version FROM skills WHERE name=?`, name).Scan(&desc, &version); e != nil {
|
||||
return x, e
|
||||
}
|
||||
} else if e := s.db.QueryRowContext(ctx, `SELECT description FROM skills WHERE name=?`, name).Scan(&desc); e != nil {
|
||||
return x, e
|
||||
}
|
||||
if e := s.db.QueryRowContext(ctx, `SELECT digest,source_path,imported_at FROM skill_versions WHERE name=? AND version=?`, name, version).Scan(&x.Digest, &x.SourcePath, &x.ImportedAt); e != nil {
|
||||
return x, e
|
||||
}
|
||||
x.Name = name
|
||||
x.Description = desc
|
||||
x.Version = version
|
||||
x.LatestVersion = version
|
||||
x.SkillMD, _ = s.skillMD(name, version)
|
||||
x.Manifest, _ = s.skillManifest(ctx, name, version)
|
||||
return x, nil
|
||||
}
|
||||
func (s *Store) skillMD(name string, v int) (string, error) {
|
||||
b, e := os.ReadFile(filepath.Join(s.dir, "skills", name, fmt.Sprintf("v%d", v), "SKILL.md"))
|
||||
return string(b), e
|
||||
}
|
||||
func (s *Store) skillManifest(ctx context.Context, name string, v int) ([]protocol.SkillFileMeta, error) {
|
||||
rows, e := s.db.QueryContext(ctx, `SELECT path,size FROM skill_files WHERE name=? AND version=? ORDER BY path`, name, v)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []protocol.SkillFileMeta
|
||||
for rows.Next() {
|
||||
var x protocol.SkillFileMeta
|
||||
if e = rows.Scan(&x.Path, &x.Size); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
out = append(out, x)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func digestSkill(files []protocol.SkillFile) string {
|
||||
h := sha256.New()
|
||||
for _, f := range files {
|
||||
fmt.Fprintf(h, "%s\x00%d\x00", f.Path, len(f.Bytes))
|
||||
h.Write(f.Bytes)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
func TestSkillImportDeduplicatesAndVersions(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "pipelines.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
files := []protocol.SkillFile{{Path: "SKILL.md", Bytes: []byte("---\nname: demo\ndescription: Demo\n---\n")}}
|
||||
first, err := s.ImportSkill(context.Background(), "demo", "Demo", digestSkill(files), "/one", files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
same, err := s.ImportSkill(context.Background(), "demo", "Demo", digestSkill(files), "/two", files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if same.Version != first.Version {
|
||||
t.Fatalf("dedup version = %d, want %d", same.Version, first.Version)
|
||||
}
|
||||
files[0].Bytes = append(files[0].Bytes, '\n')
|
||||
second, err := s.ImportSkill(context.Background(), "demo", "Demo", digestSkill(files), "/three", files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.Version != 2 {
|
||||
t.Fatalf("new version = %d, want 2", second.Version)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkillImportRejectsUnsafeName(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "pipelines.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
files := []protocol.SkillFile{{Path: "SKILL.md", Bytes: []byte("skill")}}
|
||||
if _, err := s.ImportSkill(context.Background(), "../../outside", "", digestSkill(files), "/source", files); err == nil {
|
||||
t.Fatal("expected unsafe skill name to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkillImportConcurrentVersionsAreDistinct(t *testing.T) {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "pipelines.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
const n = 8
|
||||
var wg sync.WaitGroup
|
||||
versions := make([]int, n)
|
||||
errs := make([]error, n)
|
||||
for i := range n {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
files := []protocol.SkillFile{{Path: "SKILL.md", Bytes: []byte(fmt.Sprintf("---\nname: demo\ndescription: Demo %d\n---\n", i))}}
|
||||
v, err := s.ImportSkill(context.Background(), "demo", "Demo", digestSkill(files), "/src", files)
|
||||
versions[i], errs[i] = v.Version, err
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
seen := map[int]bool{}
|
||||
for i := range n {
|
||||
if errs[i] != nil {
|
||||
t.Fatalf("import %d: %v", i, errs[i])
|
||||
}
|
||||
if seen[versions[i]] {
|
||||
t.Fatalf("version %d allocated twice: %v", versions[i], versions)
|
||||
}
|
||||
seen[versions[i]] = true
|
||||
}
|
||||
for v := 1; v <= n; v++ {
|
||||
if !seen[v] {
|
||||
t.Fatalf("versions = %v, want 1..%d", versions, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -129,6 +129,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := s.initSkills(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,12 @@ const (
|
||||
PipelineArchived = "pipeline_archived"
|
||||
UnarchivePipeline = "unarchive_pipeline"
|
||||
PipelineUnarchived = "pipeline_unarchived"
|
||||
ImportSkill = "import_skill"
|
||||
SkillImported = "skill_imported"
|
||||
ListSkills = "list_skills"
|
||||
Skills = "skills"
|
||||
ShowSkill = "show_skill"
|
||||
Skill = "skill"
|
||||
)
|
||||
|
||||
type Stage struct {
|
||||
@@ -127,6 +133,9 @@ type PipelineSummary struct {
|
||||
// are optional so older peers ignore them. See search.go for the per-type payloads
|
||||
// and the ranking/kind/bound vocabulary.
|
||||
type Message struct {
|
||||
SkillFiles []SkillFile `json:"skill_files,omitempty"`
|
||||
SkillSummaries []SkillSummary `json:"skill_summaries,omitempty"`
|
||||
SkillVersion *SkillVersion `json:"skill_version,omitempty"`
|
||||
Storage *StorageOverview `json:"storage,omitempty"`
|
||||
StoragePolicy *StoragePolicy `json:"storage_policy,omitempty"`
|
||||
StorageCleanup *StorageCleanupResult `json:"storage_cleanup,omitempty"`
|
||||
@@ -154,6 +163,9 @@ type Message struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
Warnings []string `json:"warnings,omitempty"` // non-blocking authoring lint for validate/submit
|
||||
Name string `json:"name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
SourcePath string `json:"source_path,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Version int `json:"version,omitempty"`
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package protocol
|
||||
|
||||
type SkillFile struct {
|
||||
Path string `json:"path"`
|
||||
Bytes []byte `json:"bytes"`
|
||||
}
|
||||
type SkillSummary struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
LatestVersion int `json:"latest_version"`
|
||||
Digest string `json:"digest"`
|
||||
ImportedAt string `json:"imported_at"`
|
||||
}
|
||||
type SkillVersion struct {
|
||||
SkillSummary
|
||||
Version int `json:"version"`
|
||||
SourcePath string `json:"source_path"`
|
||||
Manifest []SkillFileMeta `json:"manifest"`
|
||||
SkillMD string `json:"skill_md"`
|
||||
}
|
||||
type SkillFileMeta struct {
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
"github.com/coder/websocket/wsjson"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/skills"
|
||||
)
|
||||
|
||||
func TestSkillWebSocketHandlers(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "skills.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
handler := Handler(ctx, store)
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
|
||||
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
var welcome protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
if err := writeSkillFixture(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bundle, err := skills.Validate(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files := make([]protocol.SkillFile, len(bundle.Files))
|
||||
for i, file := range bundle.Files {
|
||||
files[i] = protocol.SkillFile{Path: file.Path, Bytes: file.Bytes}
|
||||
}
|
||||
request := protocol.Message{Type: protocol.ImportSkill, ID: "import", Name: bundle.Name, Description: bundle.Description, Digest: bundle.Digest, SourcePath: dir, SkillFiles: files}
|
||||
response := skillRequest(t, ctx, conn, request)
|
||||
if response.Type != protocol.SkillImported || response.SkillVersion == nil || response.SkillVersion.Version != 1 {
|
||||
t.Fatalf("import response = %+v", response)
|
||||
}
|
||||
|
||||
response = skillRequest(t, ctx, conn, protocol.Message{Type: protocol.ListSkills, ID: "list"})
|
||||
if response.Type != protocol.Skills || len(response.SkillSummaries) != 1 || response.SkillSummaries[0].Name != bundle.Name {
|
||||
t.Fatalf("list response = %+v", response)
|
||||
}
|
||||
response = skillRequest(t, ctx, conn, protocol.Message{Type: protocol.ShowSkill, ID: "show", Name: bundle.Name})
|
||||
if response.Type != protocol.Skill || response.SkillVersion == nil || response.SkillVersion.SkillMD == "" {
|
||||
t.Fatalf("show response = %+v", response)
|
||||
}
|
||||
|
||||
response = skillRequest(t, ctx, conn, protocol.Message{Type: protocol.ImportSkill, ID: "bad", Name: bundle.Name, SourcePath: dir, Digest: bundle.Digest, SkillFiles: []protocol.SkillFile{{Path: "../escape", Bytes: []byte("x")}}})
|
||||
if response.Type != protocol.Error || response.Error == "" {
|
||||
t.Fatalf("invalid import response = %+v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func skillRequest(t *testing.T, ctx context.Context, conn *websocket.Conn, request protocol.Message) protocol.Message {
|
||||
t.Helper()
|
||||
if err := wsjson.Write(ctx, conn, request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var response protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.ID != request.ID {
|
||||
t.Fatalf("response ID = %q, want %q", response.ID, request.ID)
|
||||
}
|
||||
return response
|
||||
}
|
||||
|
||||
func writeSkillFixture(dir string) error {
|
||||
return os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte("---\nname: websocket-skill\ndescription: Handler test\n---\n\nDo not execute this.\n"), 0600)
|
||||
}
|
||||
@@ -461,6 +461,45 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
} else {
|
||||
response.Pipelines = items
|
||||
}
|
||||
} else if request.Type == protocol.ImportSkill && len(stores) > 0 {
|
||||
response.Type = protocol.SkillImported
|
||||
if len(request.SkillFiles) == 0 || request.Name == "" || request.SourcePath == "" {
|
||||
response.Type = protocol.Error
|
||||
response.Error = "skill import requires name, source path and files"
|
||||
} else {
|
||||
for _, f := range request.SkillFiles {
|
||||
if f.Path == "" || strings.HasPrefix(f.Path, "/") || strings.Contains(f.Path, "..") {
|
||||
response.Type = protocol.Error
|
||||
response.Error = "invalid skill file path"
|
||||
break
|
||||
}
|
||||
}
|
||||
if response.Type != protocol.Error {
|
||||
var skill protocol.SkillVersion
|
||||
skill, err = stores[0].ImportSkill(ctx, request.Name, request.Description, request.Digest, request.SourcePath, request.SkillFiles)
|
||||
response.SkillVersion = &skill
|
||||
}
|
||||
if err != nil {
|
||||
response.Type = protocol.Error
|
||||
response.Error = err.Error()
|
||||
}
|
||||
}
|
||||
} else if request.Type == protocol.ListSkills && len(stores) > 0 {
|
||||
response.Type = protocol.Skills
|
||||
response.SkillSummaries, err = stores[0].ListSkills(ctx)
|
||||
if err != nil {
|
||||
response.Type = protocol.Error
|
||||
response.Error = err.Error()
|
||||
}
|
||||
} else if request.Type == protocol.ShowSkill && len(stores) > 0 {
|
||||
response.Type = protocol.Skill
|
||||
var skill protocol.SkillVersion
|
||||
skill, err = stores[0].GetSkill(ctx, request.Name, request.Version)
|
||||
response.SkillVersion = &skill
|
||||
if err != nil {
|
||||
response.Type = protocol.Error
|
||||
response.Error = err.Error()
|
||||
}
|
||||
} else if (request.Type == protocol.ArchivePipeline || request.Type == protocol.UnarchivePipeline) && len(stores) > 0 {
|
||||
detail, err := archivePipeline(ctx, stores[0], request)
|
||||
response.Detail = detail
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
// Package skills validates and fingerprints Claude/Codex/OpenCode/Pi skill bundles.
|
||||
package skills
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxFiles = 200
|
||||
MaxTotalBytes = 5 * 1024 * 1024
|
||||
MaxFileBytes = 1024 * 1024
|
||||
)
|
||||
|
||||
var slug = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`)
|
||||
|
||||
// ValidateName checks the server-facing skill name before it is used in a
|
||||
// filesystem path.
|
||||
func ValidateName(name string) error {
|
||||
if !slug.MatchString(name) {
|
||||
return fmt.Errorf("skill name must be a lowercase slug")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type File struct {
|
||||
Path string
|
||||
Bytes []byte
|
||||
}
|
||||
type Bundle struct {
|
||||
Name, Description, Digest string
|
||||
Files []File
|
||||
TotalBytes int64
|
||||
}
|
||||
|
||||
func Validate(root string) (Bundle, error) {
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return Bundle{}, err
|
||||
}
|
||||
st, err := os.Lstat(root)
|
||||
if err != nil {
|
||||
return Bundle{}, err
|
||||
}
|
||||
if st.Mode()&os.ModeSymlink != 0 {
|
||||
return Bundle{}, fmt.Errorf("symlinks are not allowed: %s", root)
|
||||
}
|
||||
if !st.IsDir() {
|
||||
return Bundle{}, fmt.Errorf("skill path is not a directory")
|
||||
}
|
||||
var files []File
|
||||
err = filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
rel, e := filepath.Rel(root, path)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
if d.Type()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("symlinks are not allowed: %s", rel)
|
||||
}
|
||||
if d.IsDir() {
|
||||
if filepath.Dir(rel) == "." && rel != "references" && rel != "scripts" && rel != "assets" {
|
||||
return fmt.Errorf("unsupported top-level directory: %s", rel)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
clean := filepath.ToSlash(rel)
|
||||
if filepath.Dir(rel) == "." && clean != "SKILL.md" {
|
||||
return fmt.Errorf("unsupported top-level file: %s", clean)
|
||||
}
|
||||
if filepath.IsAbs(rel) || strings.HasPrefix(clean, "../") || clean == ".." {
|
||||
return fmt.Errorf("invalid path: %s", rel)
|
||||
}
|
||||
if filepath.Base(rel) == "SKILL.md" && filepath.Dir(rel) != "." {
|
||||
return fmt.Errorf("nested SKILL.md is not allowed: %s", rel)
|
||||
}
|
||||
if len(files) >= MaxFiles {
|
||||
return fmt.Errorf("skill has more than %d files", MaxFiles)
|
||||
}
|
||||
info, e := d.Info()
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if info.Size() > MaxFileBytes {
|
||||
return fmt.Errorf("file %s exceeds %d bytes", clean, MaxFileBytes)
|
||||
}
|
||||
b, e := os.ReadFile(path)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if info.Mode()&0111 != 0 && !strings.HasPrefix(clean, "scripts/") {
|
||||
return fmt.Errorf("executable outside scripts/: %s", clean)
|
||||
}
|
||||
files = append(files, File{Path: clean, Bytes: b})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return Bundle{}, err
|
||||
}
|
||||
sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path })
|
||||
var md []byte
|
||||
for _, f := range files {
|
||||
if f.Path == "SKILL.md" {
|
||||
md = f.Bytes
|
||||
break
|
||||
}
|
||||
}
|
||||
if md == nil {
|
||||
return Bundle{}, fmt.Errorf("skill must contain exactly one top-level SKILL.md")
|
||||
}
|
||||
name, desc, err := frontmatter(md)
|
||||
if err != nil {
|
||||
return Bundle{}, err
|
||||
}
|
||||
if err := ValidateName(name); err != nil {
|
||||
return Bundle{}, fmt.Errorf("frontmatter name must be a lowercase slug")
|
||||
}
|
||||
var total int64
|
||||
h := sha256.New()
|
||||
for _, f := range files {
|
||||
total += int64(len(f.Bytes))
|
||||
fmt.Fprintf(h, "%s\x00%d\x00", f.Path, len(f.Bytes))
|
||||
h.Write(f.Bytes)
|
||||
}
|
||||
if total > MaxTotalBytes {
|
||||
return Bundle{}, fmt.Errorf("skill exceeds %d total bytes", MaxTotalBytes)
|
||||
}
|
||||
return Bundle{Name: name, Description: desc, Digest: hex.EncodeToString(h.Sum(nil)), Files: files, TotalBytes: total}, nil
|
||||
}
|
||||
|
||||
func frontmatter(b []byte) (string, string, error) {
|
||||
s := strings.ReplaceAll(string(b), "\r\n", "\n")
|
||||
lines := strings.Split(s, "\n")
|
||||
if len(lines) < 3 || strings.TrimSpace(lines[0]) != "---" {
|
||||
return "", "", fmt.Errorf("SKILL.md must begin with YAML frontmatter")
|
||||
}
|
||||
name, desc := "", ""
|
||||
end := -1
|
||||
for i := 1; i < len(lines); i++ {
|
||||
if strings.TrimSpace(lines[i]) == "---" {
|
||||
end = i
|
||||
break
|
||||
}
|
||||
k, v, ok := strings.Cut(lines[i], ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
v = strings.TrimSpace(v)
|
||||
v = strings.Trim(v, "\"'")
|
||||
switch strings.TrimSpace(k) {
|
||||
case "name":
|
||||
name = v
|
||||
case "description":
|
||||
desc = v
|
||||
}
|
||||
}
|
||||
if end < 0 || name == "" || desc == "" {
|
||||
return "", "", fmt.Errorf("SKILL.md frontmatter requires name and description")
|
||||
}
|
||||
return name, desc, nil
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package skills
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateDigestStable(t *testing.T) {
|
||||
for _, order := range [][]string{{"b.txt", "a.txt"}, {"a.txt", "b.txt"}} {
|
||||
d := t.TempDir()
|
||||
os.WriteFile(filepath.Join(d, "SKILL.md"), []byte("---\nname: demo-skill\ndescription: Demo\n---\n"), 0600)
|
||||
os.Mkdir(filepath.Join(d, "references"), 0700)
|
||||
for _, n := range order {
|
||||
os.WriteFile(filepath.Join(d, "references", n), []byte(n), 0600)
|
||||
}
|
||||
b, e := Validate(d)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if order[0] == "b.txt" {
|
||||
t.Setenv("DIGEST", b.Digest)
|
||||
} else if os.Getenv("DIGEST") != b.Digest {
|
||||
t.Fatalf("digest changed")
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestValidateRejectsNested(t *testing.T) {
|
||||
d := t.TempDir()
|
||||
os.WriteFile(filepath.Join(d, "SKILL.md"), []byte("---\nname: x\ndescription: y\n---"), 0600)
|
||||
os.Mkdir(filepath.Join(d, "references"), 0700)
|
||||
os.WriteFile(filepath.Join(d, "references", "SKILL.md"), []byte("x"), 0600)
|
||||
if _, e := Validate(d); e == nil {
|
||||
t.Fatal("expected nested rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNameRejectsTraversal(t *testing.T) {
|
||||
for _, name := range []string{"../outside", "../../outside", "Upper", "has space", ""} {
|
||||
if err := ValidateName(name); err == nil {
|
||||
t.Errorf("ValidateName(%q) accepted unsafe name", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsSymlinkRoot(t *testing.T) {
|
||||
target := t.TempDir()
|
||||
os.WriteFile(filepath.Join(target, "SKILL.md"), []byte("---\nname: demo\ndescription: Demo\n---\n"), 0600)
|
||||
link := filepath.Join(t.TempDir(), "skill-link")
|
||||
if err := os.Symlink(target, link); err != nil {
|
||||
t.Skipf("symlinks unavailable: %v", err)
|
||||
}
|
||||
if _, err := Validate(link); err == nil {
|
||||
t.Fatal("expected symlink root rejection")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user