refactor: serve the api specs as plain json (#38715)

The api specs were Go templates whose committed form was not a valid
swagger document, so `swagger-validate`, `generate-openapi.go` and
`.spectral.yaml` each worked around it. They are now plain json,
substituted at serve time.

Renaming them off `.tmpl` also stops `make fmt` rewriting them, which
used to bump their mtime and silently skip the next `make
generate-swagger`.

Also enables stricter spectral linting: extends `lint-swagger` to the
OpenAPI 3 spec, turns on `openapi-tags`, `operation-singular-tag` and
`operation-tag-defined`, adds a top-level `tags` array with descriptions
to the swagger input, and drops the redundant `repository` tag from
`POST /user/repos`.

---------

Signed-off-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Giteabot <teabot@gitea.io>

.editorconfig

@@ -18,7 +18,7 @@ indent_style = tab
 [templates/custom/*.tmpl]
 insert_final_newline = false
 
-[templates/swagger/*_json.tmpl]
+[templates/swagger/*.generated.json]
 indent_style = space
 insert_final_newline = false
 

.gitattributes

@@ -3,8 +3,7 @@
 *.pb.go linguist-generated
 /assets/*.json linguist-generated
 /public/assets/img/svg/*.svg linguist-generated
-/templates/swagger/v1_json.tmpl linguist-generated
+/templates/swagger/*.generated.json linguist-generated
-/templates/swagger/v1_openapi3_json.tmpl linguist-generated
 /options/fileicon/** linguist-generated
 /vendor/** -text -eol linguist-vendored
 /web_src/js/vendor/** -text -eol linguist-vendored

.github/workflows/files-changed.yml

@@ -113,8 +113,7 @@ jobs:
               - "Dockerfile.rootless"
 
             swagger:
-              - "templates/swagger/v1_json.tmpl"
+              - "templates/swagger/*.json"
-              - "templates/swagger/v1_input.json"
               - "Makefile"
               - "package.json"
               - "pnpm-lock.yaml"

.spectral.yaml

@@ -4,9 +4,4 @@ rules:
   info-contact: off
   oas2-api-host: off
   oas2-parameter-description: off
-  oas2-schema: off
-  oas2-valid-schema-example: off
-  openapi-tags: off
   operation-description: off
-  operation-singular-tag: off
-  operation-tag-defined: off

Makefile

@@ -150,10 +150,10 @@ GO_SOURCES += $(GENERATED_GO_DEST)
 ESLINT_CONCURRENCY ?= 2
 ESLINT_ARGS := --color --max-warnings=0 --concurrency $(ESLINT_CONCURRENCY)
 
-SWAGGER_SPEC := templates/swagger/v1_json.tmpl
-SWAGGER_SPEC_INPUT := templates/swagger/v1_input.json
 SWAGGER_EXCLUDE := gitea.dev/sdk
-OPENAPI3_SPEC := templates/swagger/v1_openapi3_json.tmpl
+SWAGGER_SPEC_INPUT := templates/swagger/v1-input.json
+SWAGGER_SPEC := templates/swagger/v1-swagger.generated.json
+OPENAPI3_SPEC := templates/swagger/v1-openapi3.generated.json
 
 TEST_MYSQL_HOST ?= mysql:3306
 TEST_MYSQL_DBNAME ?= testgitea
@@ -247,13 +247,10 @@ swagger-check: generate-swagger
 
 .PHONY: swagger-validate
 swagger-validate: ## check if the swagger spec is valid
-	@# swagger "validate" requires that the "basePath" must start with a slash, but we are using Golang template "{{...}}"
+	@# ensure no warnings
-	@$(SED_INPLACE) -E -e 's|"basePath":( *)"(.*)"|"basePath":\1"/\2"|g' './$(SWAGGER_SPEC)' # add a prefix slash to basePath
 	@output="$$($(GO) run $(SWAGGER_PACKAGE) validate './$(SWAGGER_SPEC)' 2>&1)"; status=$$?; \
-	$(SED_INPLACE) -E -e 's|"basePath":( *)"/(.*)"|"basePath":\1"\2"|g' './$(SWAGGER_SPEC)'; \
 	printf '%s\n' "$$output" | grep -v '^go: '; \
-	[ $$status -eq 0 ] || exit $$status; \
+	case "$$output" in *WARNING:*) exit 1;; esac; exit $$status
-	case "$$output" in *WARNING:*) exit 1;; esac
 
 .PHONY: generate-openapi3
 generate-openapi3: $(OPENAPI3_SPEC) ## generate the OpenAPI 3.0 spec from the Swagger 2.0 spec
@@ -317,7 +314,7 @@ lint-css-fix: node_modules ## lint css files and fix issues
 
 .PHONY: lint-swagger
 lint-swagger: node_modules ## lint swagger files
-	pnpm exec spectral lint -q -F hint $(SWAGGER_SPEC)
+	pnpm exec spectral lint -q -F hint $(SWAGGER_SPEC) $(OPENAPI3_SPEC)
 
 .PHONY: lint-md
 lint-md: node_modules ## lint markdown files

build/generate-openapi.go

@@ -10,7 +10,7 @@
 // cleaner SDK output with proper enum types instead of anonymous strings.
 //
 // Run: go run build/generate-openapi.go
-// Output: templates/swagger/v1_openapi3_json.tmpl
+// Output: templates/swagger/v1-openapi3.generated.json
 
 //go:build ignore
 
@@ -21,35 +21,21 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"regexp"
 	"sort"
 	"strings"
 
 	"gitea.dev/build/openapi3gen"
-
-	"github.com/getkin/kin-openapi/openapi3"
 )
 
 const (
-	swaggerSpecPath = "templates/swagger/v1_json.tmpl"
+	swaggerSpecPath = "templates/swagger/v1-swagger.generated.json"
-	openapi3OutPath = "templates/swagger/v1_openapi3_json.tmpl"
+	openapi3OutPath = "templates/swagger/v1-openapi3.generated.json"
-
-	appSubUrlVar = "{{.SwaggerAppSubUrl}}"
-	appVerVar    = "{{.SwaggerAppVer}}"
-
-	appSubUrlPlaceholder = "GITEA_APP_SUB_URL_PLACEHOLDER"
-	appVerPlaceholder    = "0.0.0-gitea-placeholder"
 )
 
-var (
+var enumScanDirs = []string{
-	appSubUrlRe = regexp.MustCompile(regexp.QuoteMeta(appSubUrlVar))
+	"modules/structs",
-	appVerRe    = regexp.MustCompile(regexp.QuoteMeta(appVerVar))
+	"modules/commitstatus",
-
+}
-	enumScanDirs = []string{
-		"modules/structs",
-		"modules/commitstatus",
-	}
-)
 
 func main() {
 	astEnumMap, err := openapi3gen.ScanSwaggerEnumTypes(enumScanDirs)
@@ -68,28 +54,17 @@ func main() {
 		log.Fatalf("reading swagger spec: %v", err)
 	}
 
-	cleaned := appSubUrlRe.ReplaceAll(data, []byte(appSubUrlPlaceholder))
+	oas3, err := openapi3gen.Convert(data, astEnumMap)
-	cleaned = appVerRe.ReplaceAll(cleaned, []byte(appVerPlaceholder))
-
-	oas3, err := openapi3gen.Convert(cleaned, astEnumMap)
 	if err != nil {
 		log.Fatalf("converting to openapi 3.0: %v", err)
 	}
 
-	oas3.Servers = openapi3.Servers{
-		{URL: appSubUrlPlaceholder + "/api/v1"},
-	}
-
 	out, err := json.MarshalIndent(oas3, "", "  ")
 	if err != nil {
 		log.Fatalf("marshaling openapi 3.0: %v", err)
 	}
 
-	result := strings.ReplaceAll(string(out), appSubUrlPlaceholder, appSubUrlVar)
+	if err := os.WriteFile(openapi3OutPath, out, 0o644); err != nil {
-	result = strings.ReplaceAll(result, appVerPlaceholder, appVerVar)
-	result = strings.TrimSpace(result)
-
-	if err := os.WriteFile(openapi3OutPath, []byte(result), 0o644); err != nil {
 		log.Fatalf("writing openapi 3.0 spec: %v", err)
 	}
 

build/openapi3gen/convert.go

@@ -23,8 +23,8 @@ import (
 var rxDeprecated = regexp.MustCompile(`(?i)(?:^|[\n.;])\s*deprecated\b`)
 
 // Convert parses a Swagger 2.0 spec and returns an OAS3 spec, applying
-// Gitea-specific post-processing: file-schema fixups, URI formats,
+// Gitea-specific post-processing: server URL, file-schema fixups, URI
-// deprecated flags, and shared-enum extraction.
+// formats, deprecated flags, and shared-enum extraction.
 //
 // astEnumMap is a value-set-key → Go-type-name(s) map (built by
 // ScanSwaggerEnumTypes). When a value set is shared by multiple Go types,
@@ -42,6 +42,7 @@ func Convert(swaggerJSON []byte, astEnumMap map[string][]string) (*openapi3.T, e
 		return nil, fmt.Errorf("converting to openapi 3.0: %w", err)
 	}
 
+	oas3.Servers = openapi3.Servers{{URL: swagger2.BasePath}}
 	fixFileSchemas(oas3)
 	addURIFormats(oas3)
 	addDeprecatedFlags(oas3)

routers/api/v1/repo/repo.go

@@ -277,7 +277,7 @@ func CreateUserRepo(ctx *context.APIContext, owner *user_model.User, opt api.Cre
 
 // Create one repository of mine
 func Create(ctx *context.APIContext) {
-	// swagger:operation POST /user/repos repository user createCurrentUserRepo
+	// swagger:operation POST /user/repos user createCurrentUserRepo
 	// ---
 	// summary: Create a repository
 	// consumes:

routers/web/auth/oauth2_provider.go

@@ -482,6 +482,7 @@ func OIDCWellKnown(ctx *context.Context) {
 	ctx.Data["OidcIssuer"] = jwtRegisteredClaims.Issuer // use the consistent issuer from the JWT registered claims
 	ctx.Data["OidcBaseUrl"] = strings.TrimSuffix(setting.AppURL, "/")
 	ctx.Data["SigningKeyMethodAlg"] = oauth2_provider.DefaultSigningKey.SigningMethod().Alg()
+	// FIXME: no need to use a Golang template to render JSON, just build the JSON response directly in the future
 	ctx.JSONTemplate("user/auth/oidc_wellknown")
 }
 

routers/web/swagger_json.go

@@ -5,21 +5,33 @@ package web
 
 import (
 	"html/template"
+	"net/http"
+	"strings"
 
 	"gitea.dev/modules/setting"
+	"gitea.dev/modules/templates"
+	"gitea.dev/modules/util"
 	"gitea.dev/services/context"
 )
 
-// SwaggerV1Json render swagger v1 json
+func swaggerJsonServe(ctx *context.Context, file string) {
-func SwaggerV1Json(ctx *context.Context) {
+	buf, err := templates.AssetFS().ReadFile(file)
-	ctx.Data["SwaggerAppVer"] = template.HTML(template.JSEscapeString(setting.AppVer))
+	if err != nil {
-	ctx.Data["SwaggerAppSubUrl"] = setting.AppSubURL // it is JS-safe
+		ctx.HTTPError(http.StatusInternalServerError, "unable to read api json file: "+file)
-	ctx.JSONTemplate("swagger/v1_json")
+		return
+	}
+	r := strings.NewReplacer(
+		"0.0.0+GITEA-API-APP-VERSION", template.JSEscapeString(setting.AppVer),
+		"/GITEA-API-APP-SUBURL/", template.JSEscapeString(setting.AppSubURL)+"/",
+	)
+	ctx.Resp.Header().Set("Content-Type", "application/json")
+	_, _ = r.WriteString(ctx.Resp, util.UnsafeBytesToString(buf))
 }
 
-// OpenAPI3Json render OpenAPI 3.0 json (auto-converted from Swagger 2.0)
+func SwaggerV1Json(ctx *context.Context) {
+	swaggerJsonServe(ctx, "swagger/v1-swagger.generated.json")
+}
+
 func OpenAPI3Json(ctx *context.Context) {
-	ctx.Data["SwaggerAppVer"] = template.HTML(template.JSEscapeString(setting.AppVer))
+	swaggerJsonServe(ctx, "swagger/v1-openapi3.generated.json")
-	ctx.Data["SwaggerAppSubUrl"] = setting.AppSubURL // it is JS-safe
-	ctx.JSONTemplate("swagger/v1_openapi3_json")
 }

tailwind.config.ts

@@ -26,7 +26,6 @@ export default {
   prefix: 'tw-',
   important: true, // the frameworks are mixed together, so tailwind needs to override other framework's styles
   content: [
-    '!./templates/swagger/v1_json.tmpl',
     '!./templates/user/auth/oidc_wellknown.tmpl',
     '!**/*_test.go',
     './{build,models,modules,routers,services}/**/*.go',

templates/swagger/v1-input.json

@@ -0,0 +1,17 @@
+{
+  "info": {
+    "version": "0.0.0+GITEA-API-APP-VERSION"
+  },
+  "basePath": "/GITEA-API-APP-SUBURL/api/v1",
+  "tags": [
+    {"name": "admin", "description": "Site administration"},
+    {"name": "issue", "description": "Issues, pull requests, comments, labels and milestones"},
+    {"name": "miscellaneous", "description": "Miscellaneous endpoints"},
+    {"name": "notification", "description": "User notifications"},
+    {"name": "organization", "description": "Organizations and teams"},
+    {"name": "package", "description": "Package registry"},
+    {"name": "repository", "description": "Repositories and their contents"},
+    {"name": "settings", "description": "Server settings"},
+    {"name": "user", "description": "The authenticated user"}
+  ]
+}

templates/swagger/v1-openapi3.generated.json

@@ -10733,7 +10733,7 @@
       "url": "http://opensource.org/licenses/MIT"
     },
     "title": "Gitea API",
-    "version": "{{.SwaggerAppVer}}"
+    "version": "0.0.0+GITEA-API-APP-VERSION"
   },
   "openapi": "3.0.3",
   "paths": {
@@ -33073,7 +33073,6 @@
         },
         "summary": "Create a repository",
         "tags": [
-          "repository",
           "user"
         ]
       }
@@ -34201,7 +34200,45 @@
   ],
   "servers": [
     {
-      "url": "{{.SwaggerAppSubUrl}}/api/v1"
+      "url": "/GITEA-API-APP-SUBURL/api/v1"
+    }
+  ],
+  "tags": [
+    {
+      "description": "Site administration",
+      "name": "admin"
+    },
+    {
+      "description": "Issues, pull requests, comments, labels and milestones",
+      "name": "issue"
+    },
+    {
+      "description": "Miscellaneous endpoints",
+      "name": "miscellaneous"
+    },
+    {
+      "description": "User notifications",
+      "name": "notification"
+    },
+    {
+      "description": "Organizations and teams",
+      "name": "organization"
+    },
+    {
+      "description": "Package registry",
+      "name": "package"
+    },
+    {
+      "description": "Repositories and their contents",
+      "name": "repository"
+    },
+    {
+      "description": "Server settings",
+      "name": "settings"
+    },
+    {
+      "description": "The authenticated user",
+      "name": "user"
     }
   ]
 }

templates/swagger/v1-swagger.generated.json

@@ -17,9 +17,9 @@
       "name": "MIT",
       "url": "http://opensource.org/licenses/MIT"
     },
-    "version": "{{.SwaggerAppVer}}"
+    "version": "0.0.0+GITEA-API-APP-VERSION"
   },
-  "basePath": "{{.SwaggerAppSubUrl}}/api/v1",
+  "basePath": "/GITEA-API-APP-SUBURL/api/v1",
   "paths": {
     "/admin/actions/jobs": {
       "get": {
@@ -20912,7 +20912,6 @@
           "application/json"
         ],
         "tags": [
-          "repository",
           "user"
         ],
         "summary": "Create a repository",
@@ -32087,5 +32086,43 @@
     {
       "TOTPHeader": []
     }
+  ],
+  "tags": [
+    {
+      "description": "Site administration",
+      "name": "admin"
+    },
+    {
+      "description": "Issues, pull requests, comments, labels and milestones",
+      "name": "issue"
+    },
+    {
+      "description": "Miscellaneous endpoints",
+      "name": "miscellaneous"
+    },
+    {
+      "description": "User notifications",
+      "name": "notification"
+    },
+    {
+      "description": "Organizations and teams",
+      "name": "organization"
+    },
+    {
+      "description": "Package registry",
+      "name": "package"
+    },
+    {
+      "description": "Repositories and their contents",
+      "name": "repository"
+    },
+    {
+      "description": "Server settings",
+      "name": "settings"
+    },
+    {
+      "description": "The authenticated user",
+      "name": "user"
+    }
   ]
 }

templates/swagger/v1_input.json

@@ -1,6 +0,0 @@
-{
-  "info": {
-    "version": "{{.SwaggerAppVer}}"
-  },
-  "basePath": "{{.SwaggerAppSubUrl}}/api/v1"
-}

tests/integration/links_test.go

@@ -31,6 +31,42 @@ func TestLinks(t *testing.T) {
 	t.Run("NoLoginNotExist", testLinksNoLoginNotExist)
 	t.Run("AsUser", testLinksAsUser)
 	t.Run("RepoCommon", testLinksRepoCommon)
+	t.Run("ApiJson", testLinksApiJson)
+}
+
+func testLinksApiJson(t *testing.T) {
+	defer test.MockVariableValue(&setting.AppVer, "1.2.3")()
+	defer test.MockVariableValue(&setting.AppSubURL)()
+	t.Run("Swagger", func(t *testing.T) {
+		for _, subURL := range []string{"", "/sub"} {
+			setting.AppSubURL = subURL
+			resp := MakeRequest(t, NewRequest(t, "GET", "/swagger.v1.json"), http.StatusOK)
+			decoded := DecodeJSON(t, resp, &struct {
+				BasePath string `json:"basePath"`
+				Info     struct {
+					Version string `json:"version"`
+				}
+			}{})
+			assert.Equal(t, subURL+"/api/v1", decoded.BasePath)
+			assert.Equal(t, "1.2.3", decoded.Info.Version)
+		}
+	})
+	t.Run("OpenAPI3", func(t *testing.T) {
+		for _, subURL := range []string{"", "/sub"} {
+			setting.AppSubURL = subURL
+			resp := MakeRequest(t, NewRequest(t, "GET", "/openapi3.v1.json"), http.StatusOK)
+			decoded := DecodeJSON(t, resp, &struct {
+				Servers []struct {
+					URL string `json:"url"`
+				} `json:"servers"`
+				Info struct {
+					Version string `json:"version"`
+				}
+			}{})
+			assert.Equal(t, subURL+"/api/v1", decoded.Servers[0].URL)
+			assert.Equal(t, "1.2.3", decoded.Info.Version)
+		}
+	})
 }
 
 func testLinksNoLogin(t *testing.T) {