feat(licenses): support REUSE specification in licenses (#38720)

extends the current license detection to support two modes: - legacy which is using classification and was expanded to handle more paths (extensions, different spelling or GNU copying file) - REUSE which avoids classification by relying on the spec dictating that license must be named as SPDX-ID.extension. Newly created repositories will default to REUSE based paths Use of styles at the same time is not allowed by design. Extends the UI to show all the detected licenses and paths to them, deduplicating them per SPDX-ID in database as is in github closes: https://github.com/go-gitea/gitea/issues/28672 --------- Assisted-By: omp:glm5.2 Assisted-By: omp:mimo-v2.5-pro Assisted-By: omp:mimimax-m3 Assisted-By: omp:kimi-k3 Assisted-By: omp:deepseek-v4-flash Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
2026-09-08 22:13:26 +09:00 · 2026-08-03 07:34:53 +00:00
parent 09f7c71e3d
commit e23fe79e5e
10 changed files with 479 additions and 70 deletions
@@ -308,16 +308,15 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User,
 	}

 	// 6 - update licenses
-	var licenses []string
 	if len(opts.License) > 0 {
-		licenses = append(licenses, opts.License)
-
+		licenses := make([]repo_model.DetectedLicense, 0, 1)
 		var stdout string
 		stdout, _, err = gitcmd.NewCommand("rev-parse", "HEAD").WithRepo(repo).RunStdString(ctx)
 		if err != nil {
 			log.Error("CreateRepository(git rev-parse HEAD) in %v: Stdout: %s\nError: %v", repo, stdout, err)
 			return nil, fmt.Errorf("CreateRepository(git rev-parse HEAD): %w", err)
 		}
+		licenses = append(licenses, repo_model.DetectedLicense{SPDXID: opts.License, LicensePath: LicenseLegacyFile})
 		if err = repo_model.UpdateRepoLicenses(ctx, repo, stdout, licenses); err != nil {
 			return nil, err
 		}
@@ -7,6 +7,8 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"path"
+	"strings"

 	"gitea.dev/models/db"
 	repo_model "gitea.dev/models/repo"
@@ -16,13 +18,20 @@ import (
 	"gitea.dev/modules/log"
 	"gitea.dev/modules/options"
 	"gitea.dev/modules/queue"
+	"gitea.dev/modules/util"

 	licenseclassifier "github.com/google/licenseclassifier/v2"
 )

+const (
+	LicenseLegacyFile = "LICENSE"
+	// LicenseReuseDir is for REUSE license spec - see https://reuse.software/spec-3.3/
+	// TODO: Surface this version in repo creation
+	LicenseReuseDir = "LICENSES"
+)
+
 var (
-	classifier      *licenseclassifier.Classifier
-	LicenseFileName = "LICENSE"
+	classifier *licenseclassifier.Classifier

 	// licenseUpdaterQueue represents a queue to handle update repo licenses
 	licenseUpdaterQueue *queue.WorkerPoolQueue[*LicenseUpdaterOptions]
@@ -71,21 +80,23 @@ func repoLicenseUpdater(items ...*LicenseUpdaterOptions) []*LicenseUpdaterOption
 			continue
 		}

-		gitRepo, err := git.OpenRepository(ctx, repo)
-		if err != nil {
-			log.Error("repoLicenseUpdater [%d] failed: OpenRepository: %v", opts.RepoID, err)
-			continue
-		}
-		defer gitRepo.Close()
+		func() {
+			gitRepo, err := git.OpenRepository(ctx, repo)
+			if err != nil {
+				log.Error("repoLicenseUpdater [%d] failed: OpenRepository: %v", opts.RepoID, err)
+				return
+			}
+			defer gitRepo.Close()

-		commit, err := gitRepo.GetBranchCommit(ctx, repo.DefaultBranch)
-		if err != nil {
-			log.Error("repoLicenseUpdater [%d] failed: GetBranchCommit: %v", opts.RepoID, err)
-			continue
-		}
-		if err = UpdateRepoLicenses(ctx, repo, gitRepo, commit); err != nil {
-			log.Error("repoLicenseUpdater [%d] failed: updateRepoLicenses: %v", opts.RepoID, err)
-		}
+			commit, err := gitRepo.GetBranchCommit(ctx, repo.DefaultBranch)
+			if err != nil {
+				log.Error("repoLicenseUpdater [%d] failed: GetBranchCommit: %v", opts.RepoID, err)
+				return
+			}
+			if err = UpdateRepoLicenses(ctx, repo, gitRepo, commit); err != nil {
+				log.Error("repoLicenseUpdater [%d] failed: updateRepoLicenses: %v", opts.RepoID, err)
+			}
+		}()
 	}
 	return nil
 }
@@ -113,43 +124,94 @@ func SyncRepoLicenses(ctx context.Context) error {
 	return nil
 }

+// resolveReuseLicenses gathers all licenses in a subtree (assumed to be LicenseReuseDir as per REUSE specification)
+func resolveReuseLicenses(ctx context.Context, gitrepo *git.Repository, parentPath string, tree *git.Tree) ([]repo_model.DetectedLicense, error) {
+	entries, err := tree.ListEntries(ctx, gitrepo)
+	if err != nil {
+		return nil, fmt.Errorf("ListEntries: %w", err)
+	}
+	licenses := make([]repo_model.DetectedLicense, 0)
+	for _, entry := range entries {
+		if entry.IsRegular() {
+			spdxID := util.PathBaseStem(entry.Name())
+			licenses = append(licenses, repo_model.DetectedLicense{SPDXID: spdxID, LicensePath: path.Join(parentPath, entry.Name())})
+		}
+	}
+
+	return licenses, nil
+}
+
+// isLicenseFile checks the prefix of the file and determines if it could plausibly be a license one
+// it's checking well-known ones: license, licence and copying
+// allowed extensions are: md, lesser and txt
+func isLicenseFile(name string) bool {
+	lower := strings.ToLower(name)
+	stem := util.PathBaseStem(lower)
+	ext := path.Ext(lower)
+	// exact match (e.g. "LICENSE") or at most one allowed extension (e.g. "LICENSE.md")
+	return (stem == "license" || stem == "licence" || stem == "copying") &&
+		(ext == "" || ext == ".md" || ext == ".lesser" || ext == ".txt")
+}
+
+func resolveLicenses(ctx context.Context, gitRepo *git.Repository, commit *git.Commit) ([]repo_model.DetectedLicense, error) {
+	tree, err := commit.SubTree(ctx, gitRepo, LicenseReuseDir)
+	if err != nil && !git.IsErrNotExist(err) {
+		return nil, fmt.Errorf("SubTree: %w", err)
+	}
+
+	// handle REUSE license spec first
+	if !git.IsErrNotExist(err) {
+		return resolveReuseLicenses(ctx, gitRepo, LicenseReuseDir, tree)
+	}
+
+	tree, err = commit.SubTree(ctx, gitRepo, "")
+	if err != nil && !git.IsErrNotExist(err) {
+		return nil, fmt.Errorf("SubTree: %w", err)
+	}
+
+	entries, err := tree.ListEntries(ctx, gitRepo)
+	if err != nil {
+		return nil, fmt.Errorf("ListEntries: %w", err)
+	}
+	licenses := make([]repo_model.DetectedLicense, 0)
+	for _, entry := range entries {
+		if !entry.IsRegular() {
+			continue
+		}
+		if !isLicenseFile(entry.Name()) {
+			continue
+		}
+		r, err := entry.Blob(gitRepo).DataAsync(ctx)
+		if err != nil {
+			continue
+		}
+		found, err := detectLicense(r)
+		_ = r.Close()
+		if err != nil {
+			continue
+		}
+		for _, license := range found {
+			licenses = append(licenses, repo_model.DetectedLicense{SPDXID: license, LicensePath: entry.Name()})
+		}
+	}
+	return licenses, nil
+}
+
 // UpdateRepoLicenses will update repository licenses col if license file exists
 func UpdateRepoLicenses(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, commit *git.Commit) error {
-	if commit == nil {
-		return nil
+	licenses, err := resolveLicenses(ctx, gitRepo, commit)
+	if err != nil {
+		return err
 	}
-
-	b, err := commit.GetBlobByPath(ctx, gitRepo, LicenseFileName)
-	if err != nil && !git.IsErrNotExist(err) {
-		return fmt.Errorf("GetBlobByPath: %w", err)
-	}
-
-	if git.IsErrNotExist(err) {
+	if len(licenses) == 0 {
 		return repo_model.CleanRepoLicenses(ctx, repo)
 	}

-	licenses := make([]string, 0)
-	if b != nil {
-		r, err := b.DataAsync(ctx)
-		if err != nil {
-			return err
-		}
-		defer r.Close()
-
-		licenses, err = detectLicense(r)
-		if err != nil {
-			return fmt.Errorf("detectLicense: %w", err)
-		}
-	}
 	return repo_model.UpdateRepoLicenses(ctx, repo, commit.ID.String(), licenses)
 }

 // detectLicense returns the licenses detected by the given content buff
 func detectLicense(r io.Reader) ([]string, error) {
-	if r == nil {
-		return nil, nil
-	}
-
 	matches, err := classifier.MatchFrom(r)
 	if err != nil {
 		return nil, err
@@ -4,9 +4,12 @@
 package repository

 import (
+	"path/filepath"
 	"strings"
 	"testing"

+	repo_model "gitea.dev/models/repo"
+	"gitea.dev/modules/git"
 	repo_module "gitea.dev/modules/repository"

 	"github.com/stretchr/testify/assert"
@@ -41,7 +44,7 @@ func Test_detectLicense(t *testing.T) {
 			Repo:  "gitea",
 			Year:  "2024",
 		})
-		assert.NoError(t, err)
+		require.NoError(t, err)

 		tests = append(tests, DetectLicenseTest{
 			name: "single license test: " + licenseName,
@@ -59,12 +62,161 @@ func Test_detectLicense(t *testing.T) {
 		})
 	}

-	result, err := detectLicense(strings.NewReader(tests[2].arg + tests[3].arg + tests[4].arg))
+	// Build multi-license content from the first 3 real license entries.
+	require.GreaterOrEqual(t, len(repo_module.Licenses), 3, "need at least 3 licenses for multi-license test")
+	var multiContent strings.Builder
+	var multiWant []string
+	for _, name := range repo_module.Licenses[:3] {
+		lic, err := repo_module.GetLicense(name, &repo_module.LicenseValues{
+			Owner: "Gitea", Email: "teabot@gitea.io", Repo: "gitea", Year: "2024",
+		})
+		require.NoError(t, err)
+		multiContent.Write(lic)
+		multiWant = append(multiWant, name)
+	}
+	t.Run("multiple licenses", func(t *testing.T) {
+		result, err := detectLicense(strings.NewReader(multiContent.String()))
+		assert.NoError(t, err)
+		assert.ElementsMatch(t, multiWant, result)
+	})
+}
+
+func detectedLicenseIDs(licenses []repo_model.DetectedLicense) []string {
+	ids := make([]string, 0, len(licenses))
+	for _, l := range licenses {
+		ids = append(ids, l.SPDXID)
+	}
+	return ids
+}
+
+func Test_resolveLicenses(t *testing.T) {
+	require.NoError(t, repo_module.LoadRepoConfig())
+	require.NoError(t, InitLicenseClassifier())
+
+	mitLicense, err := repo_module.GetLicense("MIT", &repo_module.LicenseValues{
+		Owner: "Test",
+		Email: "test@test.com",
+		Repo:  "test",
+		Year:  "2024",
+	})
+	require.NoError(t, err)
+
+	repoDir := filepath.Join(t.TempDir(), "repo.git")
+	require.NoError(t, git.InitRepositoryLocal(t.Context(), repoDir, true, "sha1"))
+	gitRepo, err := git.OpenRepositoryLocal(t.Context(), repoDir)
+	require.NoError(t, err)
+	defer gitRepo.Close()
+
+	// 1. repo with no license at all
+	require.NoError(t, git.ForceFastImport(t.Context(), gitRepo, []git.FastImportCommit{{
+		Ref:     "refs/heads/master",
+		Message: "empty",
+	}}))
+	commit, err := gitRepo.GetBranchCommit(t.Context(), "master")
+	require.NoError(t, err)
+
+	licenses, err := resolveLicenses(t.Context(), gitRepo, commit)
+	assert.Empty(t, licenses)
 	assert.NoError(t, err)
-	t.Run("multiple licenses test", func(t *testing.T) {
-		assert.Len(t, result, 3)
-		assert.Contains(t, result, tests[2].want[0])
-		assert.Contains(t, result, tests[3].want[0])
-		assert.Contains(t, result, tests[4].want[0])
+
+	// 2. repo with a plain LICENSE file — classifier should detect MIT
+	require.NoError(t, git.ForceFastImport(t.Context(), gitRepo, []git.FastImportCommit{{
+		Ref:     "refs/heads/master",
+		Message: "add LICENSE",
+		Files: []git.FastImportFile{
+			{Mode: git.EntryModeBlob, Path: "LICENSE", Content: string(mitLicense)},
+		},
+	}}))
+	commit, err = gitRepo.GetBranchCommit(t.Context(), "master")
+	require.NoError(t, err)
+
+	licenses, err = resolveLicenses(t.Context(), gitRepo, commit)
+	assert.NoError(t, err)
+	assert.Len(t, licenses, 1)
+	assert.Equal(t, "MIT", licenses[0].SPDXID)
+	assert.Equal(t, "LICENSE", licenses[0].LicensePath)
+
+	// 3. repo with REUSE LICENSES/ dir — should take priority over root LICENSE
+	require.NoError(t, git.ForceFastImport(t.Context(), gitRepo, []git.FastImportCommit{{
+		Ref:     "refs/heads/master",
+		Message: "add LICENSES dir",
+		Files: []git.FastImportFile{
+			{Mode: git.EntryModeBlob, Path: "LICENSE", Content: string(mitLicense)},
+			{Mode: git.EntryModeBlob, Path: "LICENSES/MIT.txt", Content: "MIT license text"},
+			{Mode: git.EntryModeBlob, Path: "LICENSES/Apache-2.0.txt", Content: "Apache license text"},
+		},
+	}}))
+	commit, err = gitRepo.GetBranchCommit(t.Context(), "master")
+	require.NoError(t, err)
+
+	licenses, err = resolveLicenses(t.Context(), gitRepo, commit)
+	assert.NoError(t, err)
+	assert.Len(t, licenses, 2)
+
+	ids := detectedLicenseIDs(licenses)
+	assert.ElementsMatch(t, []string{"Apache-2.0", "MIT"}, ids)
+	// REUSE entries carry full path including LICENSES/ prefix
+	for _, l := range licenses {
+		assert.True(t, strings.HasPrefix(l.LicensePath, "LICENSES/"))
+	}
+
+	// 4. remove all licenses — should return not exist
+	require.NoError(t, git.ForceFastImport(t.Context(), gitRepo, []git.FastImportCommit{{
+		Ref:     "refs/heads/master",
+		Message: "remove licenses",
+	}}))
+	commit, err = gitRepo.GetBranchCommit(t.Context(), "master")
+	require.NoError(t, err)
+
+	licenses, err = resolveLicenses(t.Context(), gitRepo, commit)
+	assert.Empty(t, licenses)
+	assert.NoError(t, err)
+}
+
+func Test_isLicenseFile(t *testing.T) {
+	shouldMatch := []string{
+		"LICENSE",
+		"LICENCE",
+		"COPYING",
+		"License",
+		"Licence",
+		"copying",
+		"LICENSE.txt",
+		"LICENSE.md",
+		"LICENCE.txt",
+		"LICENCE.md",
+		"COPYING.txt",
+		"COPYING.md",
+		"LICENSE.TXT",
+		"LICENSE.MD",
+		"license.txt",
+		"licence.md",
+		"copying.TXT",
+		"COPYING.LESSER",
+	}
+
+	shouldNotMatch := []string{
+		"LICENSE.",
+		"README",
+		"README.md",
+		"NOTICE",
+		"AUTHORS",
+		"LICENSING",
+		"COPYLEFT",
+		"LICENSE.a.b",
+		"LICENSE.a.",
+		"LICENSE.a.md",
+		"LICENSE.md.a",
+	}
+	t.Run("match", func(t *testing.T) {
+		for _, name := range shouldMatch {
+			assert.True(t, isLicenseFile(name))
+		}
+	})
+
+	t.Run("nomatch", func(t *testing.T) {
+		for _, name := range shouldNotMatch {
+			assert.False(t, isLicenseFile(name))
+		}
 	})
 }