gitea source for verification 2026-05-22

modules/actions/artifacts.go

@@ -0,0 +1,48 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+import (
+	"net/http"
+
+	actions_model "code.gitea.io/gitea/models/actions"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
+	"code.gitea.io/gitea/services/context"
+)
+
+// Artifacts using the v4 backend are stored as a single combined zip file per artifact on the backend
+// The v4 backend ensures ContentEncoding is set to "application/zip", which is not the case for the old backend
+func IsArtifactV4(art *actions_model.ActionArtifact) bool {
+	return art.ArtifactName+".zip" == art.ArtifactPath && art.ContentEncoding == "application/zip"
+}
+
+func DownloadArtifactV4ServeDirectOnly(ctx *context.Base, art *actions_model.ActionArtifact) (bool, error) {
+	if setting.Actions.ArtifactStorage.ServeDirect() {
+		u, err := storage.ActionsArtifacts.URL(art.StoragePath, art.ArtifactPath, ctx.Req.Method, nil)
+		if u != nil && err == nil {
+			ctx.Redirect(u.String(), http.StatusFound)
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func DownloadArtifactV4Fallback(ctx *context.Base, art *actions_model.ActionArtifact) error {
+	f, err := storage.ActionsArtifacts.Open(art.StoragePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	http.ServeContent(ctx.Resp, ctx.Req, art.ArtifactName+".zip", art.CreatedUnix.AsLocalTime(), f)
+	return nil
+}
+
+func DownloadArtifactV4(ctx *context.Base, art *actions_model.ActionArtifact) error {
+	ok, err := DownloadArtifactV4ServeDirectOnly(ctx, art)
+	if ok || err != nil {
+		return err
+	}
+	return DownloadArtifactV4Fallback(ctx, art)
+}

modules/actions/github.go

@@ -0,0 +1,125 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+import (
+	webhook_module "code.gitea.io/gitea/modules/webhook"
+)
+
+const (
+	GithubEventPullRequest              = "pull_request"
+	GithubEventPullRequestTarget        = "pull_request_target"
+	GithubEventPullRequestReviewComment = "pull_request_review_comment"
+	GithubEventPullRequestReview        = "pull_request_review"
+	GithubEventRegistryPackage          = "registry_package"
+	GithubEventCreate                   = "create"
+	GithubEventDelete                   = "delete"
+	GithubEventFork                     = "fork"
+	GithubEventPush                     = "push"
+	GithubEventIssues                   = "issues"
+	GithubEventIssueComment             = "issue_comment"
+	GithubEventRelease                  = "release"
+	GithubEventPullRequestComment       = "pull_request_comment"
+	GithubEventGollum                   = "gollum"
+	GithubEventSchedule                 = "schedule"
+)
+
+// IsDefaultBranchWorkflow returns true if the event only triggers workflows on the default branch
+func IsDefaultBranchWorkflow(triggedEvent webhook_module.HookEventType) bool {
+	switch triggedEvent {
+	case webhook_module.HookEventDelete:
+		// GitHub "delete" event
+		// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#delete
+		return true
+	case webhook_module.HookEventFork:
+		// GitHub "fork" event
+		// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#fork
+		return true
+	case webhook_module.HookEventIssueComment:
+		// GitHub "issue_comment" event
+		// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#issue_comment
+		return true
+	case webhook_module.HookEventPullRequestComment:
+		// GitHub "pull_request_comment" event
+		// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_comment-use-issue_comment
+		return true
+	case webhook_module.HookEventWiki:
+		// GitHub "gollum" event
+		// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#gollum
+		return true
+	case webhook_module.HookEventSchedule:
+		// GitHub "schedule" event
+		// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule
+		return true
+	case webhook_module.HookEventIssues,
+		webhook_module.HookEventIssueAssign,
+		webhook_module.HookEventIssueLabel,
+		webhook_module.HookEventIssueMilestone:
+		// Github "issues" event
+		// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#issues
+		return true
+	}
+
+	return false
+}
+
+// canGithubEventMatch check if the input Github event can match any Gitea event.
+func canGithubEventMatch(eventName string, triggedEvent webhook_module.HookEventType) bool {
+	switch eventName {
+	case GithubEventRegistryPackage:
+		return triggedEvent == webhook_module.HookEventPackage
+
+	// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#gollum
+	case GithubEventGollum:
+		return triggedEvent == webhook_module.HookEventWiki
+
+	case GithubEventIssues:
+		switch triggedEvent {
+		case webhook_module.HookEventIssues,
+			webhook_module.HookEventIssueAssign,
+			webhook_module.HookEventIssueLabel,
+			webhook_module.HookEventIssueMilestone:
+			return true
+
+		default:
+			return false
+		}
+
+	case GithubEventPullRequest, GithubEventPullRequestTarget:
+		switch triggedEvent {
+		case webhook_module.HookEventPullRequest,
+			webhook_module.HookEventPullRequestSync,
+			webhook_module.HookEventPullRequestAssign,
+			webhook_module.HookEventPullRequestLabel,
+			webhook_module.HookEventPullRequestReviewRequest,
+			webhook_module.HookEventPullRequestMilestone:
+			return true
+
+		default:
+			return false
+		}
+
+	case GithubEventPullRequestReview:
+		switch triggedEvent {
+		case webhook_module.HookEventPullRequestReviewApproved,
+			webhook_module.HookEventPullRequestReviewComment,
+			webhook_module.HookEventPullRequestReviewRejected:
+			return true
+
+		default:
+			return false
+		}
+
+	case GithubEventSchedule:
+		return triggedEvent == webhook_module.HookEventSchedule
+
+	case GithubEventIssueComment:
+		// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_comment-use-issue_comment
+		return triggedEvent == webhook_module.HookEventIssueComment ||
+			triggedEvent == webhook_module.HookEventPullRequestComment
+
+	default:
+		return eventName == string(triggedEvent)
+	}
+}

modules/actions/github_test.go

@@ -0,0 +1,119 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+import (
+	"testing"
+
+	webhook_module "code.gitea.io/gitea/modules/webhook"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCanGithubEventMatch(t *testing.T) {
+	testCases := []struct {
+		desc           string
+		eventName      string
+		triggeredEvent webhook_module.HookEventType
+		expected       bool
+	}{
+		// registry_package event
+		{
+			"registry_package matches",
+			GithubEventRegistryPackage,
+			webhook_module.HookEventPackage,
+			true,
+		},
+		{
+			"registry_package cannot match",
+			GithubEventRegistryPackage,
+			webhook_module.HookEventPush,
+			false,
+		},
+		// issues event
+		{
+			"issue matches",
+			GithubEventIssues,
+			webhook_module.HookEventIssueLabel,
+			true,
+		},
+		{
+			"issue cannot match",
+			GithubEventIssues,
+			webhook_module.HookEventIssueComment,
+			false,
+		},
+		// issue_comment event
+		{
+			"issue_comment matches",
+			GithubEventIssueComment,
+			webhook_module.HookEventIssueComment,
+			true,
+		},
+		{
+			"issue_comment cannot match",
+			GithubEventIssueComment,
+			webhook_module.HookEventIssues,
+			false,
+		},
+		// pull_request event
+		{
+			"pull_request matches",
+			GithubEventPullRequest,
+			webhook_module.HookEventPullRequestSync,
+			true,
+		},
+		{
+			"pull_request cannot match",
+			GithubEventPullRequest,
+			webhook_module.HookEventPullRequestComment,
+			false,
+		},
+		// pull_request_target event
+		{
+			"pull_request_target matches",
+			GithubEventPullRequest,
+			webhook_module.HookEventPullRequest,
+			true,
+		},
+		{
+			"pull_request_target cannot match",
+			GithubEventPullRequest,
+			webhook_module.HookEventPullRequestComment,
+			false,
+		},
+		// pull_request_review event
+		{
+			"pull_request_review matches",
+			GithubEventPullRequestReview,
+			webhook_module.HookEventPullRequestReviewComment,
+			true,
+		},
+		{
+			"pull_request_review cannot match",
+			GithubEventPullRequestReview,
+			webhook_module.HookEventPullRequestComment,
+			false,
+		},
+		// other events
+		{
+			"create event",
+			GithubEventCreate,
+			webhook_module.HookEventCreate,
+			true,
+		},
+		{
+			"create pull request comment",
+			GithubEventIssueComment,
+			webhook_module.HookEventPullRequestComment,
+			true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			assert.Equalf(t, tc.expected, canGithubEventMatch(tc.eventName, tc.triggeredEvent), "canGithubEventMatch(%v, %v)", tc.eventName, tc.triggeredEvent)
+		})
+	}
+}

modules/actions/log.go

@@ -0,0 +1,224 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/models/dbfs"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/storage"
+	"code.gitea.io/gitea/modules/zstd"
+
+	runnerv1 "code.gitea.io/actions-proto-go/runner/v1"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+const (
+	MaxLineSize = 64 * 1024
+	DBFSPrefix  = "actions_log/"
+
+	timeFormat     = "2006-01-02T15:04:05.0000000Z07:00"
+	defaultBufSize = MaxLineSize
+)
+
+// WriteLogs appends logs to DBFS file for temporary storage.
+// It doesn't respect the file format in the filename like ".zst", since it's difficult to reopen a closed compressed file and append new content.
+// Why doesn't it store logs in object storage directly? Because it's not efficient to append content to object storage.
+func WriteLogs(ctx context.Context, filename string, offset int64, rows []*runnerv1.LogRow) ([]int, error) {
+	flag := os.O_WRONLY
+	if offset == 0 {
+		// Create file only if offset is 0, or it could result in content holes if the file doesn't exist.
+		flag |= os.O_CREATE
+	}
+	name := DBFSPrefix + filename
+	f, err := dbfs.OpenFile(ctx, name, flag)
+	if err != nil {
+		return nil, fmt.Errorf("dbfs OpenFile %q: %w", name, err)
+	}
+	defer f.Close()
+
+	stat, err := f.Stat()
+	if err != nil {
+		return nil, fmt.Errorf("dbfs Stat %q: %w", name, err)
+	}
+	if stat.Size() < offset {
+		// If the size is less than offset, refuse to write, or it could result in content holes.
+		// However, if the size is greater than offset, we can still write to overwrite the content.
+		return nil, fmt.Errorf("size of %q is less than offset", name)
+	}
+
+	if _, err := f.Seek(offset, io.SeekStart); err != nil {
+		return nil, fmt.Errorf("dbfs Seek %q: %w", name, err)
+	}
+
+	writer := bufio.NewWriterSize(f, defaultBufSize)
+
+	ns := make([]int, 0, len(rows))
+	for _, row := range rows {
+		n, err := writer.WriteString(FormatLog(row.Time.AsTime(), row.Content) + "\n")
+		if err != nil {
+			return nil, err
+		}
+		ns = append(ns, n)
+	}
+
+	if err := writer.Flush(); err != nil {
+		return nil, err
+	}
+	return ns, nil
+}
+
+func ReadLogs(ctx context.Context, inStorage bool, filename string, offset, limit int64) ([]*runnerv1.LogRow, error) {
+	f, err := OpenLogs(ctx, inStorage, filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	if _, err := f.Seek(offset, io.SeekStart); err != nil {
+		return nil, fmt.Errorf("file seek: %w", err)
+	}
+
+	scanner := bufio.NewScanner(f)
+	maxLineSize := len(timeFormat) + MaxLineSize + 1
+	scanner.Buffer(make([]byte, maxLineSize), maxLineSize)
+
+	var rows []*runnerv1.LogRow
+	for scanner.Scan() && (int64(len(rows)) < limit || limit < 0) {
+		t, c, err := ParseLog(scanner.Text())
+		if err != nil {
+			return nil, fmt.Errorf("parse log %q: %w", scanner.Text(), err)
+		}
+		rows = append(rows, &runnerv1.LogRow{
+			Time:    timestamppb.New(t),
+			Content: c,
+		})
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("ReadLogs scan: %w", err)
+	}
+
+	return rows, nil
+}
+
+const (
+	// logZstdBlockSize is the block size for zstd compression.
+	// 128KB leads the compression ratio to be close to the regular zstd compression.
+	// And it means each read from the underlying object storage will be at least 128KB*(compression ratio).
+	// The compression ratio is about 30% for text files, so the actual read size is about 38KB, which should be acceptable.
+	logZstdBlockSize = 128 * 1024 // 128KB
+)
+
+// TransferLogs transfers logs from DBFS to object storage.
+// It happens when the file is complete and no more logs will be appended.
+// It respects the file format in the filename like ".zst", and compresses the content if needed.
+func TransferLogs(ctx context.Context, filename string) (func(), error) {
+	name := DBFSPrefix + filename
+	remove := func() {
+		if err := dbfs.Remove(ctx, name); err != nil {
+			log.Warn("dbfs remove %q: %v", name, err)
+		}
+	}
+	f, err := dbfs.Open(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("dbfs open %q: %w", name, err)
+	}
+	defer f.Close()
+
+	var reader io.Reader = f
+	if strings.HasSuffix(filename, ".zst") {
+		r, w := io.Pipe()
+		reader = r
+		zstdWriter, err := zstd.NewSeekableWriter(w, logZstdBlockSize)
+		if err != nil {
+			return nil, fmt.Errorf("zstd NewSeekableWriter: %w", err)
+		}
+		go func() {
+			defer func() {
+				_ = w.CloseWithError(zstdWriter.Close())
+			}()
+			if _, err := io.Copy(zstdWriter, f); err != nil {
+				_ = w.CloseWithError(err)
+				return
+			}
+		}()
+	}
+
+	if _, err := storage.Actions.Save(filename, reader, -1); err != nil {
+		return nil, fmt.Errorf("storage save %q: %w", filename, err)
+	}
+	return remove, nil
+}
+
+func RemoveLogs(ctx context.Context, inStorage bool, filename string) error {
+	if !inStorage {
+		name := DBFSPrefix + filename
+		err := dbfs.Remove(ctx, name)
+		if err != nil {
+			return fmt.Errorf("dbfs remove %q: %w", name, err)
+		}
+		return nil
+	}
+	err := storage.Actions.Delete(filename)
+	if err != nil {
+		return fmt.Errorf("storage delete %q: %w", filename, err)
+	}
+	return nil
+}
+
+func OpenLogs(ctx context.Context, inStorage bool, filename string) (io.ReadSeekCloser, error) {
+	if !inStorage {
+		name := DBFSPrefix + filename
+		f, err := dbfs.Open(ctx, name)
+		if err != nil {
+			return nil, fmt.Errorf("dbfs open %q: %w", name, err)
+		}
+		return f, nil
+	}
+
+	f, err := storage.Actions.Open(filename)
+	if err != nil {
+		return nil, fmt.Errorf("storage open %q: %w", filename, err)
+	}
+
+	var reader io.ReadSeekCloser = f
+	if strings.HasSuffix(filename, ".zst") {
+		r, err := zstd.NewSeekableReader(f)
+		if err != nil {
+			return nil, fmt.Errorf("zstd NewSeekableReader: %w", err)
+		}
+		reader = r
+	}
+
+	return reader, nil
+}
+
+func FormatLog(timestamp time.Time, content string) string {
+	// Content shouldn't contain new line, it will break log indexes, other control chars are safe.
+	content = strings.ReplaceAll(content, "\n", `\n`)
+	if len(content) > MaxLineSize {
+		content = content[:MaxLineSize]
+	}
+	return fmt.Sprintf("%s %s", timestamp.UTC().Format(timeFormat), content)
+}
+
+func ParseLog(in string) (time.Time, string, error) {
+	index := strings.IndexRune(in, ' ')
+	if index < 0 {
+		return time.Time{}, "", fmt.Errorf("invalid log: %q", in)
+	}
+	timestamp, err := time.Parse(timeFormat, in[:index])
+	if err != nil {
+		return time.Time{}, "", err
+	}
+	return timestamp, in[index+1:], nil
+}

modules/actions/task_state.go

@@ -0,0 +1,123 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+import (
+	actions_model "code.gitea.io/gitea/models/actions"
+)
+
+const (
+	preStepName  = "Set up job"
+	postStepName = "Complete job"
+)
+
+// FullSteps returns steps with "Set up job" and "Complete job"
+func FullSteps(task *actions_model.ActionTask) []*actions_model.ActionTaskStep {
+	if len(task.Steps) == 0 {
+		return fullStepsOfEmptySteps(task)
+	}
+
+	// firstStep is the first step that has run or running, not include preStep.
+	// For example,
+	// 1. preStep(Success) -> step1(Success) -> step2(Running) -> step3(Waiting) -> postStep(Waiting): firstStep is step1.
+	// 2. preStep(Success) -> step1(Skipped) -> step2(Success) -> postStep(Success): firstStep is step2.
+	// 3. preStep(Success) -> step1(Running) -> step2(Waiting) -> postStep(Waiting): firstStep is step1.
+	// 4. preStep(Success) -> step1(Skipped) -> step2(Skipped) -> postStep(Skipped): firstStep is nil.
+	// 5. preStep(Success) -> step1(Cancelled) -> step2(Cancelled) -> postStep(Cancelled): firstStep is nil.
+	var firstStep *actions_model.ActionTaskStep
+	// lastHasRunStep is the last step that has run.
+	// For example,
+	// 1. preStep(Success) -> step1(Success) -> step2(Running) -> step3(Waiting) -> postStep(Waiting): lastHasRunStep is step1.
+	// 2. preStep(Success) -> step1(Success) -> step2(Success) -> step3(Success) -> postStep(Success): lastHasRunStep is step3.
+	// 3. preStep(Success) -> step1(Success) -> step2(Failure) -> step3 -> postStep(Waiting): lastHasRunStep is step2.
+	// So its Stopped is the Started of postStep when there are no more steps to run.
+	var lastHasRunStep *actions_model.ActionTaskStep
+
+	var logIndex int64
+	for _, step := range task.Steps {
+		if firstStep == nil && (step.Status.HasRun() || step.Status.IsRunning()) {
+			firstStep = step
+		}
+		if step.Status.HasRun() {
+			lastHasRunStep = step
+		}
+		logIndex += step.LogLength
+	}
+
+	preStep := &actions_model.ActionTaskStep{
+		Name:      preStepName,
+		LogLength: task.LogLength,
+		Started:   task.Started,
+		Status:    actions_model.StatusRunning,
+	}
+
+	// No step has run or is running, so preStep is equal to the task
+	if firstStep == nil {
+		preStep.Stopped = task.Stopped
+		preStep.Status = task.Status
+	} else {
+		preStep.LogLength = firstStep.LogIndex
+		preStep.Stopped = firstStep.Started
+		preStep.Status = actions_model.StatusSuccess
+	}
+	logIndex += preStep.LogLength
+
+	if lastHasRunStep == nil {
+		lastHasRunStep = preStep
+	}
+
+	postStep := &actions_model.ActionTaskStep{
+		Name:   postStepName,
+		Status: actions_model.StatusWaiting,
+	}
+	// If the lastHasRunStep is the last step, or it has failed, postStep has started.
+	if lastHasRunStep.Status.IsFailure() || lastHasRunStep == task.Steps[len(task.Steps)-1] {
+		postStep.LogIndex = logIndex
+		postStep.LogLength = task.LogLength - postStep.LogIndex
+		postStep.Started = lastHasRunStep.Stopped
+		postStep.Status = actions_model.StatusRunning
+	}
+	if task.Status.IsDone() {
+		postStep.Status = task.Status
+		postStep.Stopped = task.Stopped
+	}
+	ret := make([]*actions_model.ActionTaskStep, 0, len(task.Steps)+2)
+	ret = append(ret, preStep)
+	ret = append(ret, task.Steps...)
+	ret = append(ret, postStep)
+
+	return ret
+}
+
+func fullStepsOfEmptySteps(task *actions_model.ActionTask) []*actions_model.ActionTaskStep {
+	preStep := &actions_model.ActionTaskStep{
+		Name:      preStepName,
+		LogLength: task.LogLength,
+		Started:   task.Started,
+		Stopped:   task.Stopped,
+		Status:    actions_model.StatusRunning,
+	}
+
+	postStep := &actions_model.ActionTaskStep{
+		Name:     postStepName,
+		LogIndex: task.LogLength,
+		Started:  task.Stopped,
+		Stopped:  task.Stopped,
+		Status:   actions_model.StatusWaiting,
+	}
+
+	if task.Status.IsDone() {
+		preStep.Status = task.Status
+		if preStep.Status.IsSuccess() {
+			postStep.Status = actions_model.StatusSuccess
+		} else {
+			postStep.Status = actions_model.StatusCancelled
+		}
+	}
+
+	return []*actions_model.ActionTaskStep{
+		preStep,
+		postStep,
+	}
+}

modules/actions/task_state_test.go

@@ -0,0 +1,165 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+import (
+	"testing"
+
+	actions_model "code.gitea.io/gitea/models/actions"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFullSteps(t *testing.T) {
+	tests := []struct {
+		name string
+		task *actions_model.ActionTask
+		want []*actions_model.ActionTaskStep
+	}{
+		{
+			name: "regular",
+			task: &actions_model.ActionTask{
+				Steps: []*actions_model.ActionTaskStep{
+					{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
+				},
+				Status:    actions_model.StatusSuccess,
+				Started:   10000,
+				Stopped:   10100,
+				LogLength: 100,
+			},
+			want: []*actions_model.ActionTaskStep{
+				{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
+				{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
+				{Name: postStepName, Status: actions_model.StatusSuccess, LogIndex: 90, LogLength: 10, Started: 10090, Stopped: 10100},
+			},
+		},
+		{
+			name: "failed step",
+			task: &actions_model.ActionTask{
+				Steps: []*actions_model.ActionTaskStep{
+					{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 20, Started: 10010, Stopped: 10020},
+					{Status: actions_model.StatusFailure, LogIndex: 30, LogLength: 60, Started: 10020, Stopped: 10090},
+					{Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
+				},
+				Status:    actions_model.StatusFailure,
+				Started:   10000,
+				Stopped:   10100,
+				LogLength: 100,
+			},
+			want: []*actions_model.ActionTaskStep{
+				{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
+				{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 20, Started: 10010, Stopped: 10020},
+				{Status: actions_model.StatusFailure, LogIndex: 30, LogLength: 60, Started: 10020, Stopped: 10090},
+				{Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
+				{Name: postStepName, Status: actions_model.StatusFailure, LogIndex: 90, LogLength: 10, Started: 10090, Stopped: 10100},
+			},
+		},
+		{
+			name: "first step is running",
+			task: &actions_model.ActionTask{
+				Steps: []*actions_model.ActionTaskStep{
+					{Status: actions_model.StatusRunning, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 0},
+				},
+				Status:    actions_model.StatusRunning,
+				Started:   10000,
+				Stopped:   10100,
+				LogLength: 100,
+			},
+			want: []*actions_model.ActionTaskStep{
+				{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
+				{Status: actions_model.StatusRunning, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 0},
+				{Name: postStepName, Status: actions_model.StatusWaiting, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
+			},
+		},
+		{
+			name: "first step has canceled",
+			task: &actions_model.ActionTask{
+				Steps: []*actions_model.ActionTaskStep{
+					{Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
+				},
+				Status:    actions_model.StatusFailure,
+				Started:   10000,
+				Stopped:   10100,
+				LogLength: 100,
+			},
+			want: []*actions_model.ActionTaskStep{
+				{Name: preStepName, Status: actions_model.StatusFailure, LogIndex: 0, LogLength: 100, Started: 10000, Stopped: 10100},
+				{Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
+				{Name: postStepName, Status: actions_model.StatusFailure, LogIndex: 100, LogLength: 0, Started: 10100, Stopped: 10100},
+			},
+		},
+		{
+			name: "empty steps",
+			task: &actions_model.ActionTask{
+				Steps:     []*actions_model.ActionTaskStep{},
+				Status:    actions_model.StatusSuccess,
+				Started:   10000,
+				Stopped:   10100,
+				LogLength: 100,
+			},
+			want: []*actions_model.ActionTaskStep{
+				{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 100, Started: 10000, Stopped: 10100},
+				{Name: postStepName, Status: actions_model.StatusSuccess, LogIndex: 100, LogLength: 0, Started: 10100, Stopped: 10100},
+			},
+		},
+		{
+			name: "all steps finished but task is running",
+			task: &actions_model.ActionTask{
+				Steps: []*actions_model.ActionTaskStep{
+					{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
+				},
+				Status:    actions_model.StatusRunning,
+				Started:   10000,
+				Stopped:   0,
+				LogLength: 100,
+			},
+			want: []*actions_model.ActionTaskStep{
+				{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
+				{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
+				{Name: postStepName, Status: actions_model.StatusRunning, LogIndex: 90, LogLength: 10, Started: 10090, Stopped: 0},
+			},
+		},
+		{
+			name: "skipped task",
+			task: &actions_model.ActionTask{
+				Steps: []*actions_model.ActionTaskStep{
+					{Status: actions_model.StatusSkipped, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
+				},
+				Status:    actions_model.StatusSkipped,
+				Started:   0,
+				Stopped:   0,
+				LogLength: 0,
+			},
+			want: []*actions_model.ActionTaskStep{
+				{Name: preStepName, Status: actions_model.StatusSkipped, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
+				{Status: actions_model.StatusSkipped, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
+				{Name: postStepName, Status: actions_model.StatusSkipped, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
+			},
+		},
+		{
+			name: "first step is skipped",
+			task: &actions_model.ActionTask{
+				Steps: []*actions_model.ActionTaskStep{
+					{Status: actions_model.StatusSkipped, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
+					{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
+				},
+				Status:    actions_model.StatusSuccess,
+				Started:   10000,
+				Stopped:   10100,
+				LogLength: 100,
+			},
+			want: []*actions_model.ActionTaskStep{
+				{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
+				{Status: actions_model.StatusSkipped, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
+				{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
+				{Name: postStepName, Status: actions_model.StatusSuccess, LogIndex: 90, LogLength: 10, Started: 10090, Stopped: 10100},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equalf(t, tt.want, FullSteps(tt.task), "FullSteps(%v)", tt.task)
+		})
+	}
+}

modules/actions/workflows.go

@@ -0,0 +1,755 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+import (
+	"bytes"
+	"io"
+	"slices"
+	"strings"
+
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/glob"
+	"code.gitea.io/gitea/modules/log"
+	api "code.gitea.io/gitea/modules/structs"
+	webhook_module "code.gitea.io/gitea/modules/webhook"
+
+	"github.com/nektos/act/pkg/jobparser"
+	"github.com/nektos/act/pkg/model"
+	"github.com/nektos/act/pkg/workflowpattern"
+	"gopkg.in/yaml.v3"
+)
+
+type DetectedWorkflow struct {
+	EntryName    string
+	TriggerEvent *jobparser.Event
+	Content      []byte
+}
+
+func init() {
+	model.OnDecodeNodeError = func(node yaml.Node, out any, err error) {
+		// Log the error instead of panic or fatal.
+		// It will be a big job to refactor act/pkg/model to return decode error,
+		// so we just log the error and return empty value, and improve it later.
+		log.Error("Failed to decode node %v into %T: %v", node, out, err)
+	}
+}
+
+func IsWorkflow(path string) bool {
+	if (!strings.HasSuffix(path, ".yaml")) && (!strings.HasSuffix(path, ".yml")) {
+		return false
+	}
+
+	return strings.HasPrefix(path, ".gitea/workflows") || strings.HasPrefix(path, ".github/workflows")
+}
+
+func ListWorkflows(commit *git.Commit) (string, git.Entries, error) {
+	rpath := ".gitea/workflows"
+	tree, err := commit.SubTree(rpath)
+	if _, ok := err.(git.ErrNotExist); ok {
+		rpath = ".github/workflows"
+		tree, err = commit.SubTree(rpath)
+	}
+	if _, ok := err.(git.ErrNotExist); ok {
+		return "", nil, nil
+	}
+	if err != nil {
+		return "", nil, err
+	}
+
+	entries, err := tree.ListEntriesRecursiveFast()
+	if err != nil {
+		return "", nil, err
+	}
+
+	ret := make(git.Entries, 0, len(entries))
+	for _, entry := range entries {
+		if strings.HasSuffix(entry.Name(), ".yml") || strings.HasSuffix(entry.Name(), ".yaml") {
+			ret = append(ret, entry)
+		}
+	}
+	return rpath, ret, nil
+}
+
+func GetContentFromEntry(entry *git.TreeEntry) ([]byte, error) {
+	f, err := entry.Blob().DataAsync()
+	if err != nil {
+		return nil, err
+	}
+	content, err := io.ReadAll(f)
+	_ = f.Close()
+	if err != nil {
+		return nil, err
+	}
+	return content, nil
+}
+
+func GetEventsFromContent(content []byte) ([]*jobparser.Event, error) {
+	workflow, err := model.ReadWorkflow(bytes.NewReader(content))
+	if err != nil {
+		return nil, err
+	}
+	events, err := jobparser.ParseRawOn(&workflow.RawOn)
+	if err != nil {
+		return nil, err
+	}
+
+	return events, nil
+}
+
+func DetectWorkflows(
+	gitRepo *git.Repository,
+	commit *git.Commit,
+	triggedEvent webhook_module.HookEventType,
+	payload api.Payloader,
+	detectSchedule bool,
+) ([]*DetectedWorkflow, []*DetectedWorkflow, error) {
+	_, entries, err := ListWorkflows(commit)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	workflows := make([]*DetectedWorkflow, 0, len(entries))
+	schedules := make([]*DetectedWorkflow, 0, len(entries))
+	for _, entry := range entries {
+		content, err := GetContentFromEntry(entry)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// one workflow may have multiple events
+		events, err := GetEventsFromContent(content)
+		if err != nil {
+			log.Warn("ignore invalid workflow %q: %v", entry.Name(), err)
+			continue
+		}
+		for _, evt := range events {
+			log.Trace("detect workflow %q for event %#v matching %q", entry.Name(), evt, triggedEvent)
+			if evt.IsSchedule() {
+				if detectSchedule {
+					dwf := &DetectedWorkflow{
+						EntryName:    entry.Name(),
+						TriggerEvent: evt,
+						Content:      content,
+					}
+					schedules = append(schedules, dwf)
+				}
+			} else if detectMatched(gitRepo, commit, triggedEvent, payload, evt) {
+				dwf := &DetectedWorkflow{
+					EntryName:    entry.Name(),
+					TriggerEvent: evt,
+					Content:      content,
+				}
+				workflows = append(workflows, dwf)
+			}
+		}
+	}
+
+	return workflows, schedules, nil
+}
+
+func DetectScheduledWorkflows(gitRepo *git.Repository, commit *git.Commit) ([]*DetectedWorkflow, error) {
+	_, entries, err := ListWorkflows(commit)
+	if err != nil {
+		return nil, err
+	}
+
+	wfs := make([]*DetectedWorkflow, 0, len(entries))
+	for _, entry := range entries {
+		content, err := GetContentFromEntry(entry)
+		if err != nil {
+			return nil, err
+		}
+
+		// one workflow may have multiple events
+		events, err := GetEventsFromContent(content)
+		if err != nil {
+			log.Warn("ignore invalid workflow %q: %v", entry.Name(), err)
+			continue
+		}
+		for _, evt := range events {
+			if evt.IsSchedule() {
+				log.Trace("detect scheduled workflow: %q", entry.Name())
+				dwf := &DetectedWorkflow{
+					EntryName:    entry.Name(),
+					TriggerEvent: evt,
+					Content:      content,
+				}
+				wfs = append(wfs, dwf)
+			}
+		}
+	}
+
+	return wfs, nil
+}
+
+func detectMatched(gitRepo *git.Repository, commit *git.Commit, triggedEvent webhook_module.HookEventType, payload api.Payloader, evt *jobparser.Event) bool {
+	if !canGithubEventMatch(evt.Name, triggedEvent) {
+		return false
+	}
+
+	switch triggedEvent {
+	case // events with no activity types
+		webhook_module.HookEventCreate,
+		webhook_module.HookEventDelete,
+		webhook_module.HookEventFork,
+		webhook_module.HookEventWiki,
+		webhook_module.HookEventSchedule:
+		if len(evt.Acts()) != 0 {
+			log.Warn("Ignore unsupported %s event arguments %v", triggedEvent, evt.Acts())
+		}
+		// no special filter parameters for these events, just return true if name matched
+		return true
+
+	case // push
+		webhook_module.HookEventPush:
+		return matchPushEvent(commit, payload.(*api.PushPayload), evt)
+
+	case // issues
+		webhook_module.HookEventIssues,
+		webhook_module.HookEventIssueAssign,
+		webhook_module.HookEventIssueLabel,
+		webhook_module.HookEventIssueMilestone:
+		return matchIssuesEvent(payload.(*api.IssuePayload), evt)
+
+	case // issue_comment
+		webhook_module.HookEventIssueComment,
+		// `pull_request_comment` is same as `issue_comment`
+		// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_comment-use-issue_comment
+		webhook_module.HookEventPullRequestComment:
+		return matchIssueCommentEvent(payload.(*api.IssueCommentPayload), evt)
+
+	case // pull_request
+		webhook_module.HookEventPullRequest,
+		webhook_module.HookEventPullRequestSync,
+		webhook_module.HookEventPullRequestAssign,
+		webhook_module.HookEventPullRequestLabel,
+		webhook_module.HookEventPullRequestReviewRequest,
+		webhook_module.HookEventPullRequestMilestone:
+		return matchPullRequestEvent(gitRepo, commit, payload.(*api.PullRequestPayload), evt)
+
+	case // pull_request_review
+		webhook_module.HookEventPullRequestReviewApproved,
+		webhook_module.HookEventPullRequestReviewRejected:
+		return matchPullRequestReviewEvent(payload.(*api.PullRequestPayload), evt)
+
+	case // pull_request_review_comment
+		webhook_module.HookEventPullRequestReviewComment:
+		return matchPullRequestReviewCommentEvent(payload.(*api.PullRequestPayload), evt)
+
+	case // release
+		webhook_module.HookEventRelease:
+		return matchReleaseEvent(payload.(*api.ReleasePayload), evt)
+
+	case // registry_package
+		webhook_module.HookEventPackage:
+		return matchPackageEvent(payload.(*api.PackagePayload), evt)
+
+	case // workflow_run
+		webhook_module.HookEventWorkflowRun:
+		return matchWorkflowRunEvent(payload.(*api.WorkflowRunPayload), evt)
+
+	default:
+		log.Warn("unsupported event %q", triggedEvent)
+		return false
+	}
+}
+
+func matchPushEvent(commit *git.Commit, pushPayload *api.PushPayload, evt *jobparser.Event) bool {
+	// with no special filter parameters
+	if len(evt.Acts()) == 0 {
+		return true
+	}
+
+	matchTimes := 0
+	hasBranchFilter := false
+	hasTagFilter := false
+	refName := git.RefName(pushPayload.Ref)
+	// all acts conditions should be satisfied
+	for cond, vals := range evt.Acts() {
+		switch cond {
+		case "branches":
+			hasBranchFilter = true
+			if !refName.IsBranch() {
+				break
+			}
+			patterns, err := workflowpattern.CompilePatterns(vals...)
+			if err != nil {
+				break
+			}
+			if !workflowpattern.Skip(patterns, []string{refName.BranchName()}, &workflowpattern.EmptyTraceWriter{}) {
+				matchTimes++
+			}
+		case "branches-ignore":
+			hasBranchFilter = true
+			if !refName.IsBranch() {
+				break
+			}
+			patterns, err := workflowpattern.CompilePatterns(vals...)
+			if err != nil {
+				break
+			}
+			if !workflowpattern.Filter(patterns, []string{refName.BranchName()}, &workflowpattern.EmptyTraceWriter{}) {
+				matchTimes++
+			}
+		case "tags":
+			hasTagFilter = true
+			if !refName.IsTag() {
+				break
+			}
+			patterns, err := workflowpattern.CompilePatterns(vals...)
+			if err != nil {
+				break
+			}
+			if !workflowpattern.Skip(patterns, []string{refName.TagName()}, &workflowpattern.EmptyTraceWriter{}) {
+				matchTimes++
+			}
+		case "tags-ignore":
+			hasTagFilter = true
+			if !refName.IsTag() {
+				break
+			}
+			patterns, err := workflowpattern.CompilePatterns(vals...)
+			if err != nil {
+				break
+			}
+			if !workflowpattern.Filter(patterns, []string{refName.TagName()}, &workflowpattern.EmptyTraceWriter{}) {
+				matchTimes++
+			}
+		case "paths":
+			if refName.IsTag() {
+				matchTimes++
+				break
+			}
+			filesChanged, err := commit.GetFilesChangedSinceCommit(pushPayload.Before)
+			if err != nil {
+				log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err)
+			} else {
+				patterns, err := workflowpattern.CompilePatterns(vals...)
+				if err != nil {
+					break
+				}
+				if !workflowpattern.Skip(patterns, filesChanged, &workflowpattern.EmptyTraceWriter{}) {
+					matchTimes++
+				}
+			}
+		case "paths-ignore":
+			if refName.IsTag() {
+				matchTimes++
+				break
+			}
+			filesChanged, err := commit.GetFilesChangedSinceCommit(pushPayload.Before)
+			if err != nil {
+				log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err)
+			} else {
+				patterns, err := workflowpattern.CompilePatterns(vals...)
+				if err != nil {
+					break
+				}
+				if !workflowpattern.Filter(patterns, filesChanged, &workflowpattern.EmptyTraceWriter{}) {
+					matchTimes++
+				}
+			}
+		default:
+			log.Warn("push event unsupported condition %q", cond)
+		}
+	}
+	// if both branch and tag filter are defined in the workflow only one needs to match
+	if hasBranchFilter && hasTagFilter {
+		matchTimes++
+	}
+	return matchTimes == len(evt.Acts())
+}
+
+func matchIssuesEvent(issuePayload *api.IssuePayload, evt *jobparser.Event) bool {
+	// with no special filter parameters
+	if len(evt.Acts()) == 0 {
+		return true
+	}
+
+	matchTimes := 0
+	// all acts conditions should be satisfied
+	for cond, vals := range evt.Acts() {
+		switch cond {
+		case "types":
+			// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#issues
+			// Actions with the same name:
+			// opened, edited, closed, reopened, assigned, unassigned, milestoned, demilestoned
+			// Actions need to be converted:
+			// label_updated -> labeled (when adding) or unlabeled (when removing)
+			// label_cleared -> unlabeled
+			// Unsupported activity types:
+			// deleted, transferred, pinned, unpinned, locked, unlocked
+
+			actions := []string{}
+			switch issuePayload.Action {
+			case api.HookIssueLabelUpdated:
+				if len(issuePayload.Changes.AddedLabels) > 0 {
+					actions = append(actions, "labeled")
+				}
+				if len(issuePayload.Changes.RemovedLabels) > 0 {
+					actions = append(actions, "unlabeled")
+				}
+			case api.HookIssueLabelCleared:
+				actions = append(actions, "unlabeled")
+			default:
+				actions = append(actions, string(issuePayload.Action))
+			}
+
+			for _, val := range vals {
+				if slices.ContainsFunc(actions, glob.MustCompile(val, '/').Match) {
+					matchTimes++
+					break
+				}
+			}
+		default:
+			log.Warn("issue event unsupported condition %q", cond)
+		}
+	}
+	return matchTimes == len(evt.Acts())
+}
+
+func matchPullRequestEvent(gitRepo *git.Repository, commit *git.Commit, prPayload *api.PullRequestPayload, evt *jobparser.Event) bool {
+	acts := evt.Acts()
+	activityTypeMatched := false
+	matchTimes := 0
+
+	if vals, ok := acts["types"]; !ok {
+		// defaultly, only pull request `opened`, `reopened` and `synchronized` will trigger workflow
+		// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
+		activityTypeMatched = prPayload.Action == api.HookIssueSynchronized || prPayload.Action == api.HookIssueOpened || prPayload.Action == api.HookIssueReOpened
+	} else {
+		// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
+		// Actions with the same name:
+		// opened, edited, closed, reopened, assigned, unassigned, review_requested, review_request_removed, milestoned, demilestoned
+		// Actions need to be converted:
+		// synchronized -> synchronize
+		// label_updated -> labeled
+		// label_cleared -> unlabeled
+		// Unsupported activity types:
+		// converted_to_draft, ready_for_review, locked, unlocked, auto_merge_enabled, auto_merge_disabled, enqueued, dequeued
+
+		action := prPayload.Action
+		switch action {
+		case api.HookIssueSynchronized:
+			action = "synchronize"
+		case api.HookIssueLabelUpdated:
+			action = "labeled"
+		case api.HookIssueLabelCleared:
+			action = "unlabeled"
+		}
+		log.Trace("matching pull_request %s with %v", action, vals)
+		for _, val := range vals {
+			if glob.MustCompile(val, '/').Match(string(action)) {
+				activityTypeMatched = true
+				matchTimes++
+				break
+			}
+		}
+	}
+
+	var (
+		headCommit = commit
+		err        error
+	)
+	if evt.Name == GithubEventPullRequestTarget && (len(acts["paths"]) > 0 || len(acts["paths-ignore"]) > 0) {
+		headCommit, err = gitRepo.GetCommit(prPayload.PullRequest.Head.Sha)
+		if err != nil {
+			log.Error("GetCommit [ref: %s]: %v", prPayload.PullRequest.Head.Sha, err)
+			return false
+		}
+	}
+
+	// all acts conditions should be satisfied
+	for cond, vals := range acts {
+		switch cond {
+		case "types":
+			// types have been checked
+			continue
+		case "branches":
+			refName := git.RefName(prPayload.PullRequest.Base.Ref)
+			patterns, err := workflowpattern.CompilePatterns(vals...)
+			if err != nil {
+				break
+			}
+			if !workflowpattern.Skip(patterns, []string{refName.ShortName()}, &workflowpattern.EmptyTraceWriter{}) {
+				matchTimes++
+			}
+		case "branches-ignore":
+			refName := git.RefName(prPayload.PullRequest.Base.Ref)
+			patterns, err := workflowpattern.CompilePatterns(vals...)
+			if err != nil {
+				break
+			}
+			if !workflowpattern.Filter(patterns, []string{refName.ShortName()}, &workflowpattern.EmptyTraceWriter{}) {
+				matchTimes++
+			}
+		case "paths":
+			filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.MergeBase)
+			if err != nil {
+				log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", headCommit.ID.String(), err)
+			} else {
+				patterns, err := workflowpattern.CompilePatterns(vals...)
+				if err != nil {
+					break
+				}
+				if !workflowpattern.Skip(patterns, filesChanged, &workflowpattern.EmptyTraceWriter{}) {
+					matchTimes++
+				}
+			}
+		case "paths-ignore":
+			filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.MergeBase)
+			if err != nil {
+				log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", headCommit.ID.String(), err)
+			} else {
+				patterns, err := workflowpattern.CompilePatterns(vals...)
+				if err != nil {
+					break
+				}
+				if !workflowpattern.Filter(patterns, filesChanged, &workflowpattern.EmptyTraceWriter{}) {
+					matchTimes++
+				}
+			}
+		default:
+			log.Warn("pull request event unsupported condition %q", cond)
+		}
+	}
+	return activityTypeMatched && matchTimes == len(evt.Acts())
+}
+
+func matchIssueCommentEvent(issueCommentPayload *api.IssueCommentPayload, evt *jobparser.Event) bool {
+	// with no special filter parameters
+	if len(evt.Acts()) == 0 {
+		return true
+	}
+
+	matchTimes := 0
+	// all acts conditions should be satisfied
+	for cond, vals := range evt.Acts() {
+		switch cond {
+		case "types":
+			// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#issue_comment
+			// Actions with the same name:
+			// created, edited, deleted
+			// Actions need to be converted:
+			// NONE
+			// Unsupported activity types:
+			// NONE
+
+			for _, val := range vals {
+				if glob.MustCompile(val, '/').Match(string(issueCommentPayload.Action)) {
+					matchTimes++
+					break
+				}
+			}
+		default:
+			log.Warn("issue comment event unsupported condition %q", cond)
+		}
+	}
+	return matchTimes == len(evt.Acts())
+}
+
+func matchPullRequestReviewEvent(prPayload *api.PullRequestPayload, evt *jobparser.Event) bool {
+	// with no special filter parameters
+	if len(evt.Acts()) == 0 {
+		return true
+	}
+
+	matchTimes := 0
+	// all acts conditions should be satisfied
+	for cond, vals := range evt.Acts() {
+		switch cond {
+		case "types":
+			// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_review
+			// Activity types with the same name:
+			// NONE
+			// Activity types need to be converted:
+			// reviewed -> submitted
+			// reviewed -> edited
+			// Unsupported activity types:
+			// dismissed
+
+			actions := make([]string, 0)
+			if prPayload.Action == api.HookIssueReviewed {
+				// the `reviewed` HookIssueAction can match the two activity types: `submitted` and `edited`
+				// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_review
+				actions = append(actions, "submitted", "edited")
+			}
+
+			for _, val := range vals {
+				if slices.ContainsFunc(actions, glob.MustCompile(val, '/').Match) {
+					matchTimes++
+					break
+				}
+			}
+		default:
+			log.Warn("pull request review event unsupported condition %q", cond)
+		}
+	}
+	return matchTimes == len(evt.Acts())
+}
+
+func matchPullRequestReviewCommentEvent(prPayload *api.PullRequestPayload, evt *jobparser.Event) bool {
+	// with no special filter parameters
+	if len(evt.Acts()) == 0 {
+		return true
+	}
+
+	matchTimes := 0
+	// all acts conditions should be satisfied
+	for cond, vals := range evt.Acts() {
+		switch cond {
+		case "types":
+			// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_review_comment
+			// Activity types with the same name:
+			// NONE
+			// Activity types need to be converted:
+			// reviewed -> created
+			// reviewed -> edited
+			// Unsupported activity types:
+			// deleted
+
+			actions := make([]string, 0)
+			if prPayload.Action == api.HookIssueReviewed {
+				// the `reviewed` HookIssueAction can match the two activity types: `created` and `edited`
+				// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_review_comment
+				actions = append(actions, "created", "edited")
+			}
+
+			for _, val := range vals {
+				if slices.ContainsFunc(actions, glob.MustCompile(val, '/').Match) {
+					matchTimes++
+					break
+				}
+			}
+		default:
+			log.Warn("pull request review comment event unsupported condition %q", cond)
+		}
+	}
+	return matchTimes == len(evt.Acts())
+}
+
+func matchReleaseEvent(payload *api.ReleasePayload, evt *jobparser.Event) bool {
+	// with no special filter parameters
+	if len(evt.Acts()) == 0 {
+		return true
+	}
+
+	matchTimes := 0
+	// all acts conditions should be satisfied
+	for cond, vals := range evt.Acts() {
+		switch cond {
+		case "types":
+			// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#release
+			// Activity types with the same name:
+			// published
+			// Activity types need to be converted:
+			// updated -> edited
+			// Unsupported activity types:
+			// unpublished, created, deleted, prereleased, released
+
+			action := payload.Action
+			switch action {
+			case api.HookReleaseUpdated:
+				action = "edited"
+			}
+			for _, val := range vals {
+				if glob.MustCompile(val, '/').Match(string(action)) {
+					matchTimes++
+					break
+				}
+			}
+		default:
+			log.Warn("release event unsupported condition %q", cond)
+		}
+	}
+	return matchTimes == len(evt.Acts())
+}
+
+func matchPackageEvent(payload *api.PackagePayload, evt *jobparser.Event) bool {
+	// with no special filter parameters
+	if len(evt.Acts()) == 0 {
+		return true
+	}
+
+	matchTimes := 0
+	// all acts conditions should be satisfied
+	for cond, vals := range evt.Acts() {
+		switch cond {
+		case "types":
+			// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#registry_package
+			// Activity types with the same name:
+			// NONE
+			// Activity types need to be converted:
+			// created -> published
+			// Unsupported activity types:
+			// updated
+
+			action := payload.Action
+			switch action {
+			case api.HookPackageCreated:
+				action = "published"
+			}
+			for _, val := range vals {
+				if glob.MustCompile(val, '/').Match(string(action)) {
+					matchTimes++
+					break
+				}
+			}
+		default:
+			log.Warn("package event unsupported condition %q", cond)
+		}
+	}
+	return matchTimes == len(evt.Acts())
+}
+
+func matchWorkflowRunEvent(payload *api.WorkflowRunPayload, evt *jobparser.Event) bool {
+	// with no special filter parameters
+	if len(evt.Acts()) == 0 {
+		return true
+	}
+
+	matchTimes := 0
+	// all acts conditions should be satisfied
+	for cond, vals := range evt.Acts() {
+		switch cond {
+		case "types":
+			action := payload.Action
+			for _, val := range vals {
+				if glob.MustCompile(val, '/').Match(action) {
+					matchTimes++
+					break
+				}
+			}
+		case "workflows":
+			workflow := payload.Workflow
+			patterns, err := workflowpattern.CompilePatterns(vals...)
+			if err != nil {
+				break
+			}
+			if !workflowpattern.Skip(patterns, []string{workflow.Name}, &workflowpattern.EmptyTraceWriter{}) {
+				matchTimes++
+			}
+		case "branches":
+			patterns, err := workflowpattern.CompilePatterns(vals...)
+			if err != nil {
+				break
+			}
+			if !workflowpattern.Skip(patterns, []string{payload.WorkflowRun.HeadBranch}, &workflowpattern.EmptyTraceWriter{}) {
+				matchTimes++
+			}
+		case "branches-ignore":
+			patterns, err := workflowpattern.CompilePatterns(vals...)
+			if err != nil {
+				break
+			}
+			if !workflowpattern.Filter(patterns, []string{payload.WorkflowRun.HeadBranch}, &workflowpattern.EmptyTraceWriter{}) {
+				matchTimes++
+			}
+		default:
+			log.Warn("workflow run event unsupported condition %q", cond)
+		}
+	}
+	return matchTimes == len(evt.Acts())
+}

modules/actions/workflows_test.go

@@ -0,0 +1,337 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/modules/git"
+	api "code.gitea.io/gitea/modules/structs"
+	webhook_module "code.gitea.io/gitea/modules/webhook"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDetectMatched(t *testing.T) {
+	testCases := []struct {
+		desc         string
+		commit       *git.Commit
+		triggedEvent webhook_module.HookEventType
+		payload      api.Payloader
+		yamlOn       string
+		expected     bool
+	}{
+		{
+			desc:         "HookEventCreate(create) matches GithubEventCreate(create)",
+			triggedEvent: webhook_module.HookEventCreate,
+			payload:      nil,
+			yamlOn:       "on: create",
+			expected:     true,
+		},
+		{
+			desc:         "HookEventIssues(issues) `opened` action matches GithubEventIssues(issues)",
+			triggedEvent: webhook_module.HookEventIssues,
+			payload:      &api.IssuePayload{Action: api.HookIssueOpened},
+			yamlOn:       "on: issues",
+			expected:     true,
+		},
+		{
+			desc:         "HookEventIssues(issues) `milestoned` action matches GithubEventIssues(issues)",
+			triggedEvent: webhook_module.HookEventIssues,
+			payload:      &api.IssuePayload{Action: api.HookIssueMilestoned},
+			yamlOn:       "on: issues",
+			expected:     true,
+		},
+		{
+			desc:         "HookEventPullRequestSync(pull_request_sync) matches GithubEventPullRequest(pull_request)",
+			triggedEvent: webhook_module.HookEventPullRequestSync,
+			payload:      &api.PullRequestPayload{Action: api.HookIssueSynchronized},
+			yamlOn:       "on: pull_request",
+			expected:     true,
+		},
+		{
+			desc:         "HookEventPullRequest(pull_request) `label_updated` action doesn't match GithubEventPullRequest(pull_request) with no activity type",
+			triggedEvent: webhook_module.HookEventPullRequest,
+			payload:      &api.PullRequestPayload{Action: api.HookIssueLabelUpdated},
+			yamlOn:       "on: pull_request",
+			expected:     false,
+		},
+		{
+			desc:         "HookEventPullRequest(pull_request) `closed` action doesn't match GithubEventPullRequest(pull_request) with no activity type",
+			triggedEvent: webhook_module.HookEventPullRequest,
+			payload:      &api.PullRequestPayload{Action: api.HookIssueClosed},
+			yamlOn:       "on: pull_request",
+			expected:     false,
+		},
+		{
+			desc:         "HookEventPullRequest(pull_request) `closed` action doesn't match GithubEventPullRequest(pull_request) with branches",
+			triggedEvent: webhook_module.HookEventPullRequest,
+			payload: &api.PullRequestPayload{
+				Action: api.HookIssueClosed,
+				PullRequest: &api.PullRequest{
+					Base: &api.PRBranchInfo{},
+				},
+			},
+			yamlOn:   "on:\n  pull_request:\n    branches: [main]",
+			expected: false,
+		},
+		{
+			desc:         "HookEventPullRequest(pull_request) `label_updated` action matches GithubEventPullRequest(pull_request) with `label` activity type",
+			triggedEvent: webhook_module.HookEventPullRequest,
+			payload:      &api.PullRequestPayload{Action: api.HookIssueLabelUpdated},
+			yamlOn:       "on:\n  pull_request:\n    types: [labeled]",
+			expected:     true,
+		},
+		{
+			desc:         "HookEventPullRequestReviewComment(pull_request_review_comment) matches GithubEventPullRequestReviewComment(pull_request_review_comment)",
+			triggedEvent: webhook_module.HookEventPullRequestReviewComment,
+			payload:      &api.PullRequestPayload{Action: api.HookIssueReviewed},
+			yamlOn:       "on:\n  pull_request_review_comment:\n    types: [created]",
+			expected:     true,
+		},
+		{
+			desc:         "HookEventPullRequestReviewRejected(pull_request_review_rejected) doesn't match GithubEventPullRequestReview(pull_request_review) with `dismissed` activity type (we don't support `dismissed` at present)",
+			triggedEvent: webhook_module.HookEventPullRequestReviewRejected,
+			payload:      &api.PullRequestPayload{Action: api.HookIssueReviewed},
+			yamlOn:       "on:\n  pull_request_review:\n    types: [dismissed]",
+			expected:     false,
+		},
+		{
+			desc:         "HookEventRelease(release) `published` action matches GithubEventRelease(release) with `published` activity type",
+			triggedEvent: webhook_module.HookEventRelease,
+			payload:      &api.ReleasePayload{Action: api.HookReleasePublished},
+			yamlOn:       "on:\n  release:\n    types: [published]",
+			expected:     true,
+		},
+		{
+			desc:         "HookEventPackage(package) `created` action doesn't match GithubEventRegistryPackage(registry_package) with `updated` activity type",
+			triggedEvent: webhook_module.HookEventPackage,
+			payload:      &api.PackagePayload{Action: api.HookPackageCreated},
+			yamlOn:       "on:\n  registry_package:\n    types: [updated]",
+			expected:     false,
+		},
+		{
+			desc:         "HookEventWiki(wiki) matches GithubEventGollum(gollum)",
+			triggedEvent: webhook_module.HookEventWiki,
+			payload:      nil,
+			yamlOn:       "on: gollum",
+			expected:     true,
+		},
+		{
+			desc:         "HookEventSchedue(schedule) matches GithubEventSchedule(schedule)",
+			triggedEvent: webhook_module.HookEventSchedule,
+			payload:      nil,
+			yamlOn:       "on: schedule",
+			expected:     true,
+		},
+		{
+			desc:         "push to tag matches workflow with paths condition (should skip paths check)",
+			triggedEvent: webhook_module.HookEventPush,
+			payload: &api.PushPayload{
+				Ref:    "refs/tags/v1.0.0",
+				Before: "0000000",
+				Commits: []*api.PayloadCommit{
+					{
+						ID:      "abcdef123456",
+						Added:   []string{"src/main.go"},
+						Message: "Release v1.0.0",
+					},
+				},
+			},
+			commit:   nil,
+			yamlOn:   "on:\n  push:\n    paths:\n      - src/**",
+			expected: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			evts, err := GetEventsFromContent([]byte(tc.yamlOn))
+			assert.NoError(t, err)
+			assert.Len(t, evts, 1)
+			assert.Equal(t, tc.expected, detectMatched(nil, tc.commit, tc.triggedEvent, tc.payload, evts[0]))
+		})
+	}
+}
+
+func TestMatchIssuesEvent(t *testing.T) {
+	testCases := []struct {
+		desc      string
+		payload   *api.IssuePayload
+		yamlOn    string
+		expected  bool
+		eventType string
+	}{
+		{
+			desc: "Label deletion should trigger unlabeled event",
+			payload: &api.IssuePayload{
+				Action: api.HookIssueLabelUpdated,
+				Issue: &api.Issue{
+					Labels: []*api.Label{},
+				},
+				Changes: &api.ChangesPayload{
+					RemovedLabels: []*api.Label{
+						{ID: 123, Name: "deleted-label"},
+					},
+				},
+			},
+			yamlOn:    "on:\n  issues:\n    types: [unlabeled]",
+			expected:  true,
+			eventType: "unlabeled",
+		},
+		{
+			desc: "Label deletion with existing labels should trigger unlabeled event",
+			payload: &api.IssuePayload{
+				Action: api.HookIssueLabelUpdated,
+				Issue: &api.Issue{
+					Labels: []*api.Label{
+						{ID: 456, Name: "existing-label"},
+					},
+				},
+				Changes: &api.ChangesPayload{
+					AddedLabels: nil,
+					RemovedLabels: []*api.Label{
+						{ID: 123, Name: "deleted-label"},
+					},
+				},
+			},
+			yamlOn:    "on:\n  issues:\n    types: [unlabeled]",
+			expected:  true,
+			eventType: "unlabeled",
+		},
+		{
+			desc: "Label addition should trigger labeled event",
+			payload: &api.IssuePayload{
+				Action: api.HookIssueLabelUpdated,
+				Issue: &api.Issue{
+					Labels: []*api.Label{
+						{ID: 123, Name: "new-label"},
+					},
+				},
+				Changes: &api.ChangesPayload{
+					AddedLabels: []*api.Label{
+						{ID: 123, Name: "new-label"},
+					},
+					RemovedLabels: []*api.Label{}, // Empty array, no labels removed
+				},
+			},
+			yamlOn:    "on:\n  issues:\n    types: [labeled]",
+			expected:  true,
+			eventType: "labeled",
+		},
+		{
+			desc: "Label clear should trigger unlabeled event",
+			payload: &api.IssuePayload{
+				Action: api.HookIssueLabelCleared,
+				Issue: &api.Issue{
+					Labels: []*api.Label{},
+				},
+			},
+			yamlOn:    "on:\n  issues:\n    types: [unlabeled]",
+			expected:  true,
+			eventType: "unlabeled",
+		},
+		{
+			desc: "Both adding and removing labels should trigger labeled event",
+			payload: &api.IssuePayload{
+				Action: api.HookIssueLabelUpdated,
+				Issue: &api.Issue{
+					Labels: []*api.Label{
+						{ID: 789, Name: "new-label"},
+					},
+				},
+				Changes: &api.ChangesPayload{
+					AddedLabels: []*api.Label{
+						{ID: 789, Name: "new-label"},
+					},
+					RemovedLabels: []*api.Label{
+						{ID: 123, Name: "deleted-label"},
+					},
+				},
+			},
+			yamlOn:    "on:\n  issues:\n    types: [labeled]",
+			expected:  true,
+			eventType: "labeled",
+		},
+		{
+			desc: "Both adding and removing labels should trigger unlabeled event",
+			payload: &api.IssuePayload{
+				Action: api.HookIssueLabelUpdated,
+				Issue: &api.Issue{
+					Labels: []*api.Label{
+						{ID: 789, Name: "new-label"},
+					},
+				},
+				Changes: &api.ChangesPayload{
+					AddedLabels: []*api.Label{
+						{ID: 789, Name: "new-label"},
+					},
+					RemovedLabels: []*api.Label{
+						{ID: 123, Name: "deleted-label"},
+					},
+				},
+			},
+			yamlOn:    "on:\n  issues:\n    types: [unlabeled]",
+			expected:  true,
+			eventType: "unlabeled",
+		},
+		{
+			desc: "Both adding and removing labels should trigger both events",
+			payload: &api.IssuePayload{
+				Action: api.HookIssueLabelUpdated,
+				Issue: &api.Issue{
+					Labels: []*api.Label{
+						{ID: 789, Name: "new-label"},
+					},
+				},
+				Changes: &api.ChangesPayload{
+					AddedLabels: []*api.Label{
+						{ID: 789, Name: "new-label"},
+					},
+					RemovedLabels: []*api.Label{
+						{ID: 123, Name: "deleted-label"},
+					},
+				},
+			},
+			yamlOn:    "on:\n  issues:\n    types: [labeled, unlabeled]",
+			expected:  true,
+			eventType: "multiple",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			evts, err := GetEventsFromContent([]byte(tc.yamlOn))
+			assert.NoError(t, err)
+			assert.Len(t, evts, 1)
+
+			// Test if the event matches as expected
+			assert.Equal(t, tc.expected, matchIssuesEvent(tc.payload, evts[0]))
+
+			// For extra validation, check that action mapping works correctly
+			if tc.eventType == "multiple" {
+				// Skip direct action mapping validation for multiple events case
+				// as one action can map to multiple event types
+				return
+			}
+
+			// Determine expected action for single event case
+			var expectedAction string
+			switch tc.payload.Action {
+			case api.HookIssueLabelUpdated:
+				if tc.eventType == "labeled" {
+					expectedAction = "labeled"
+				} else if tc.eventType == "unlabeled" {
+					expectedAction = "unlabeled"
+				}
+			case api.HookIssueLabelCleared:
+				expectedAction = "unlabeled"
+			default:
+				expectedAction = string(tc.payload.Action)
+			}
+
+			assert.Equal(t, expectedAction, tc.eventType, "Event type should match expected")
+		})
+	}
+}

modules/activitypub/client.go

@@ -0,0 +1,124 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package activitypub
+
+import (
+	"bytes"
+	"context"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/proxy"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/42wim/httpsig"
+)
+
+const (
+	// ActivityStreamsContentType const
+	ActivityStreamsContentType = `application/ld+json; profile="https://www.w3.org/ns/activitystreams"`
+	httpsigExpirationTime      = 60
+)
+
+// Gets the current time as an RFC 2616 formatted string
+// RFC 2616 requires RFC 1123 dates but with GMT instead of UTC
+func CurrentTime() string {
+	return strings.ReplaceAll(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT")
+}
+
+func containsRequiredHTTPHeaders(method string, headers []string) error {
+	var hasRequestTarget, hasDate, hasDigest bool
+	for _, header := range headers {
+		hasRequestTarget = hasRequestTarget || header == httpsig.RequestTarget
+		hasDate = hasDate || header == "Date"
+		hasDigest = hasDigest || header == "Digest"
+	}
+	if !hasRequestTarget {
+		return fmt.Errorf("missing http header for %s: %s", method, httpsig.RequestTarget)
+	} else if !hasDate {
+		return fmt.Errorf("missing http header for %s: Date", method)
+	} else if !hasDigest && method != http.MethodGet {
+		return fmt.Errorf("missing http header for %s: Digest", method)
+	}
+	return nil
+}
+
+// Client struct
+type Client struct {
+	client      *http.Client
+	algs        []httpsig.Algorithm
+	digestAlg   httpsig.DigestAlgorithm
+	getHeaders  []string
+	postHeaders []string
+	priv        *rsa.PrivateKey
+	pubID       string
+}
+
+// NewClient function
+func NewClient(ctx context.Context, user *user_model.User, pubID string) (c *Client, err error) {
+	if err = containsRequiredHTTPHeaders(http.MethodGet, setting.Federation.GetHeaders); err != nil {
+		return nil, err
+	} else if err = containsRequiredHTTPHeaders(http.MethodPost, setting.Federation.PostHeaders); err != nil {
+		return nil, err
+	}
+
+	priv, err := GetPrivateKey(ctx, user)
+	if err != nil {
+		return nil, err
+	}
+	privPem, _ := pem.Decode([]byte(priv))
+	privParsed, err := x509.ParsePKCS1PrivateKey(privPem.Bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	c = &Client{
+		client: &http.Client{
+			Transport: &http.Transport{
+				Proxy: proxy.Proxy(),
+			},
+		},
+		algs:        setting.HttpsigAlgs,
+		digestAlg:   httpsig.DigestAlgorithm(setting.Federation.DigestAlgorithm),
+		getHeaders:  setting.Federation.GetHeaders,
+		postHeaders: setting.Federation.PostHeaders,
+		priv:        privParsed,
+		pubID:       pubID,
+	}
+	return c, err
+}
+
+// NewRequest function
+func (c *Client) NewRequest(b []byte, to string) (req *http.Request, err error) {
+	buf := bytes.NewBuffer(b)
+	req, err = http.NewRequest(http.MethodPost, to, buf)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Content-Type", ActivityStreamsContentType)
+	req.Header.Add("Date", CurrentTime())
+	req.Header.Add("User-Agent", "Gitea/"+setting.AppVer)
+	signer, _, err := httpsig.NewSigner(c.algs, c.digestAlg, c.postHeaders, httpsig.Signature, httpsigExpirationTime)
+	if err != nil {
+		return nil, err
+	}
+	err = signer.SignRequest(c.priv, c.pubID, req, b)
+	return req, err
+}
+
+// Post function
+func (c *Client) Post(b []byte, to string) (resp *http.Response, err error) {
+	var req *http.Request
+	if req, err = c.NewRequest(b, to); err != nil {
+		return nil, err
+	}
+	resp, err = c.client.Do(req)
+	return resp, err
+}

modules/activitypub/client_test.go

@@ -0,0 +1,45 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package activitypub
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"code.gitea.io/gitea/models/unittest"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestActivityPubSignedPost(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
+	pubID := "https://example.com/pubID"
+	c, err := NewClient(t.Context(), user, pubID)
+	assert.NoError(t, err)
+
+	expected := "BODY"
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Regexp(t, "^"+setting.Federation.DigestAlgorithm, r.Header.Get("Digest"))
+		assert.Contains(t, r.Header.Get("Signature"), pubID)
+		assert.Equal(t, ActivityStreamsContentType, r.Header.Get("Content-Type"))
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
+		assert.Equal(t, expected, string(body))
+		fmt.Fprint(w, expected)
+	}))
+	defer srv.Close()
+
+	r, err := c.Post([]byte(expected), srv.URL)
+	assert.NoError(t, err)
+	defer r.Body.Close()
+	body, err := io.ReadAll(r.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, expected, string(body))
+}

modules/activitypub/main_test.go

@@ -0,0 +1,18 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package activitypub
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/unittest"
+
+	_ "code.gitea.io/gitea/models"
+	_ "code.gitea.io/gitea/models/actions"
+	_ "code.gitea.io/gitea/models/activities"
+)
+
+func TestMain(m *testing.M) {
+	unittest.MainTest(m)
+}

modules/activitypub/user_settings.go

@@ -0,0 +1,48 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package activitypub
+
+import (
+	"context"
+
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/util"
+)
+
+const rsaBits = 3072
+
+// GetKeyPair function returns a user's private and public keys
+func GetKeyPair(ctx context.Context, user *user_model.User) (pub, priv string, err error) {
+	var settings map[string]*user_model.Setting
+	settings, err = user_model.GetSettings(ctx, user.ID, []string{user_model.UserActivityPubPrivPem, user_model.UserActivityPubPubPem})
+	if err != nil {
+		return pub, priv, err
+	} else if len(settings) == 0 {
+		if priv, pub, err = util.GenerateKeyPair(rsaBits); err != nil {
+			return pub, priv, err
+		}
+		if err = user_model.SetUserSetting(ctx, user.ID, user_model.UserActivityPubPrivPem, priv); err != nil {
+			return pub, priv, err
+		}
+		if err = user_model.SetUserSetting(ctx, user.ID, user_model.UserActivityPubPubPem, pub); err != nil {
+			return pub, priv, err
+		}
+		return pub, priv, err
+	}
+	priv = settings[user_model.UserActivityPubPrivPem].SettingValue
+	pub = settings[user_model.UserActivityPubPubPem].SettingValue
+	return pub, priv, err
+}
+
+// GetPublicKey function returns a user's public key
+func GetPublicKey(ctx context.Context, user *user_model.User) (pub string, err error) {
+	pub, _, err = GetKeyPair(ctx, user)
+	return pub, err
+}
+
+// GetPrivateKey function returns a user's private key
+func GetPrivateKey(ctx context.Context, user *user_model.User) (priv string, err error) {
+	_, priv, err = GetKeyPair(ctx, user)
+	return priv, err
+}

modules/activitypub/user_settings_test.go

@@ -0,0 +1,28 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package activitypub
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/unittest"
+	user_model "code.gitea.io/gitea/models/user"
+
+	_ "code.gitea.io/gitea/models" // https://forum.gitea.com/t/testfixtures-could-not-clean-table-access-no-such-table-access/4137/4
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUserSettings(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+	user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
+	pub, priv, err := GetKeyPair(t.Context(), user1)
+	assert.NoError(t, err)
+	pub1, err := GetPublicKey(t.Context(), user1)
+	assert.NoError(t, err)
+	assert.Equal(t, pub, pub1)
+	priv1, err := GetPrivateKey(t.Context(), user1)
+	assert.NoError(t, err)
+	assert.Equal(t, priv, priv1)
+}

modules/analyze/code_language.go

@@ -0,0 +1,27 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package analyze
+
+import (
+	"path/filepath"
+
+	"github.com/go-enry/go-enry/v2"
+)
+
+// GetCodeLanguage detects code language based on file name and content
+func GetCodeLanguage(filename string, content []byte) string {
+	if language, ok := enry.GetLanguageByExtension(filename); ok {
+		return language
+	}
+
+	if language, ok := enry.GetLanguageByFilename(filename); ok {
+		return language
+	}
+
+	if len(content) == 0 {
+		return enry.OtherLanguage
+	}
+
+	return enry.GetLanguage(filepath.Base(filename), content)
+}

modules/analyze/generated.go

@@ -0,0 +1,27 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package analyze
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/go-enry/go-enry/v2/data"
+)
+
+// IsGenerated returns whether or not path is a generated path.
+func IsGenerated(path string) bool {
+	ext := strings.ToLower(filepath.Ext(path))
+	if _, ok := data.GeneratedCodeExtensions[ext]; ok {
+		return true
+	}
+
+	for _, m := range data.GeneratedCodeNameMatchers {
+		if m(path) {
+			return true
+		}
+	}
+
+	return false
+}

modules/analyze/vendor.go

@@ -0,0 +1,13 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package analyze
+
+import (
+	"github.com/go-enry/go-enry/v2"
+)
+
+// IsVendor returns whether or not path is a vendor path.
+func IsVendor(path string) bool {
+	return enry.IsVendor(path)
+}

modules/analyze/vendor_test.go

@@ -0,0 +1,44 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package analyze
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsVendor(t *testing.T) {
+	tests := []struct {
+		path string
+		want bool
+	}{
+		{"cache/", true},
+		{"random/cache/", true},
+		{"cache", false},
+		{"dependencies/", true},
+		{"Dependencies/", true},
+		{"dependency/", false},
+		{"dist/", true},
+		{"dist", false},
+		{"random/dist/", true},
+		{"random/dist", false},
+		{"deps/", true},
+		{"configure", true},
+		{"a/configure", true},
+		{"config.guess", true},
+		{"config.guess/", false},
+		{".vscode/", true},
+		{"doc/_build/", true},
+		{"a/docs/_build/", true},
+		{"a/dasdocs/_build-vsdoc.js", true},
+		{"a/dasdocs/_build-vsdoc.j", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.path, func(t *testing.T) {
+			got := IsVendor(tt.path)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

modules/assetfs/embed.go

@@ -0,0 +1,375 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package assetfs
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/util"
+)
+
+type EmbeddedFile interface {
+	io.ReadSeeker
+	fs.ReadDirFile
+	ReadDir(n int) ([]fs.DirEntry, error)
+}
+
+type EmbeddedFileInfo interface {
+	fs.FileInfo
+	fs.DirEntry
+	GetGzipContent() ([]byte, bool)
+}
+
+type decompressor interface {
+	io.Reader
+	Close() error
+	Reset(io.Reader) error
+}
+
+type embeddedFileInfo struct {
+	fs       *embeddedFS
+	fullName string
+	data     []byte
+
+	BaseName   string              `json:"n"`
+	OriginSize int64               `json:"s,omitempty"`
+	DataBegin  int64               `json:"b,omitempty"`
+	DataLen    int64               `json:"l,omitempty"`
+	Children   []*embeddedFileInfo `json:"c,omitempty"`
+}
+
+func (fi *embeddedFileInfo) GetGzipContent() ([]byte, bool) {
+	// when generating the bindata, if the compressed data equals or is larger than the original data, we store the original data
+	if fi.DataLen == fi.OriginSize {
+		return nil, false
+	}
+	return fi.data, true
+}
+
+type EmbeddedFileBase struct {
+	info       *embeddedFileInfo
+	dataReader io.ReadSeeker
+	seekPos    int64
+}
+
+func (f *EmbeddedFileBase) ReadDir(n int) ([]fs.DirEntry, error) {
+	// this method is used to satisfy the "func (f ioFile) ReadDir(...)" in httpfs
+	l, err := f.info.fs.ReadDir(f.info.fullName)
+	if err != nil {
+		return nil, err
+	}
+	if n < 0 || n > len(l) {
+		return l, nil
+	}
+	return l[:n], nil
+}
+
+type EmbeddedOriginFile struct {
+	EmbeddedFileBase
+}
+
+type EmbeddedCompressedFile struct {
+	EmbeddedFileBase
+	decompressor    decompressor
+	decompressorPos int64
+}
+
+type embeddedFS struct {
+	meta func() *EmbeddedMeta
+
+	files   map[string]*embeddedFileInfo
+	filesMu sync.RWMutex
+
+	data []byte
+}
+
+type EmbeddedMeta struct {
+	Root *embeddedFileInfo
+}
+
+func NewEmbeddedFS(data []byte) fs.ReadDirFS {
+	efs := &embeddedFS{data: data, files: make(map[string]*embeddedFileInfo)}
+	efs.meta = sync.OnceValue(func() *EmbeddedMeta {
+		var meta EmbeddedMeta
+		p := bytes.LastIndexByte(data, '\n')
+		if p < 0 {
+			return &meta
+		}
+		if err := json.Unmarshal(data[p+1:], &meta); err != nil {
+			panic("embedded file is not valid")
+		}
+		return &meta
+	})
+	return efs
+}
+
+var _ fs.ReadDirFS = (*embeddedFS)(nil)
+
+func (e *embeddedFS) ReadDir(name string) (l []fs.DirEntry, err error) {
+	fi, err := e.getFileInfo(name)
+	if err != nil {
+		return nil, err
+	}
+	if !fi.IsDir() {
+		return nil, fs.ErrNotExist
+	}
+	l = make([]fs.DirEntry, len(fi.Children))
+	for i, child := range fi.Children {
+		l[i], err = e.getFileInfo(name + "/" + child.BaseName)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return l, nil
+}
+
+func (e *embeddedFS) getFileInfo(fullName string) (*embeddedFileInfo, error) {
+	// no need to do heavy "path.Clean()" because we don't want to support "foo/../bar" or absolute paths
+	fullName = strings.TrimPrefix(fullName, "./")
+	if fullName == "" {
+		fullName = "."
+	}
+
+	e.filesMu.RLock()
+	fi := e.files[fullName]
+	e.filesMu.RUnlock()
+	if fi != nil {
+		return fi, nil
+	}
+
+	fields := strings.Split(fullName, "/")
+	fi = e.meta().Root
+	if fullName != "." {
+		found := true
+		for _, field := range fields {
+			for _, child := range fi.Children {
+				if found = child.BaseName == field; found {
+					fi = child
+					break
+				}
+			}
+			if !found {
+				return nil, fs.ErrNotExist
+			}
+		}
+	}
+
+	e.filesMu.Lock()
+	defer e.filesMu.Unlock()
+	if fi != nil {
+		fi.fs = e
+		fi.fullName = fullName
+		fi.data = e.data[fi.DataBegin : fi.DataBegin+fi.DataLen]
+		e.files[fullName] = fi // do not cache nil, otherwise keeping accessing random non-existing file will cause OOM
+		return fi, nil
+	}
+	return nil, fs.ErrNotExist
+}
+
+func (e *embeddedFS) Open(name string) (fs.File, error) {
+	info, err := e.getFileInfo(name)
+	if err != nil {
+		return nil, err
+	}
+	base := EmbeddedFileBase{info: info}
+	base.dataReader = bytes.NewReader(base.info.data)
+	if info.DataLen != info.OriginSize {
+		decomp, err := gzip.NewReader(base.dataReader)
+		if err != nil {
+			return nil, err
+		}
+		return &EmbeddedCompressedFile{EmbeddedFileBase: base, decompressor: decomp}, nil
+	}
+	return &EmbeddedOriginFile{base}, nil
+}
+
+var (
+	_ EmbeddedFileInfo = (*embeddedFileInfo)(nil)
+	_ EmbeddedFile     = (*EmbeddedOriginFile)(nil)
+	_ EmbeddedFile     = (*EmbeddedCompressedFile)(nil)
+)
+
+func (f *EmbeddedOriginFile) Read(p []byte) (n int, err error) {
+	return f.dataReader.Read(p)
+}
+
+func (f *EmbeddedCompressedFile) Read(p []byte) (n int, err error) {
+	if f.decompressorPos > f.seekPos {
+		if err = f.decompressor.Reset(bytes.NewReader(f.info.data)); err != nil {
+			return 0, err
+		}
+		f.decompressorPos = 0
+	}
+	if f.decompressorPos < f.seekPos {
+		if _, err = io.CopyN(io.Discard, f.decompressor, f.seekPos-f.decompressorPos); err != nil {
+			return 0, err
+		}
+		f.decompressorPos = f.seekPos
+	}
+	n, err = f.decompressor.Read(p)
+	f.decompressorPos += int64(n)
+	f.seekPos = f.decompressorPos
+	return n, err
+}
+
+func (f *EmbeddedFileBase) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case io.SeekStart:
+		f.seekPos = offset
+	case io.SeekCurrent:
+		f.seekPos += offset
+	case io.SeekEnd:
+		f.seekPos = f.info.OriginSize + offset
+	}
+	return f.seekPos, nil
+}
+
+func (f *EmbeddedFileBase) Stat() (fs.FileInfo, error) {
+	return f.info, nil
+}
+
+func (f *EmbeddedOriginFile) Close() error {
+	return nil
+}
+
+func (f *EmbeddedCompressedFile) Close() error {
+	return f.decompressor.Close()
+}
+
+func (fi *embeddedFileInfo) Name() string {
+	return fi.BaseName
+}
+
+func (fi *embeddedFileInfo) Size() int64 {
+	return fi.OriginSize
+}
+
+func (fi *embeddedFileInfo) Mode() fs.FileMode {
+	return util.Iif(fi.IsDir(), fs.ModeDir|0o555, 0o444)
+}
+
+func (fi *embeddedFileInfo) ModTime() time.Time {
+	return getExecutableModTime()
+}
+
+func (fi *embeddedFileInfo) IsDir() bool {
+	return fi.Children != nil
+}
+
+func (fi *embeddedFileInfo) Sys() any {
+	return nil
+}
+
+func (fi *embeddedFileInfo) Type() fs.FileMode {
+	return util.Iif(fi.IsDir(), fs.ModeDir, 0)
+}
+
+func (fi *embeddedFileInfo) Info() (fs.FileInfo, error) {
+	return fi, nil
+}
+
+// getExecutableModTime returns the modification time of the executable file.
+// In bindata, we can't use the ModTime of the files because we need to make the build reproducible
+var getExecutableModTime = sync.OnceValue(func() (modTime time.Time) {
+	exePath, err := os.Executable()
+	if err != nil {
+		return modTime
+	}
+	exePath, err = filepath.Abs(exePath)
+	if err != nil {
+		return modTime
+	}
+	exePath, err = filepath.EvalSymlinks(exePath)
+	if err != nil {
+		return modTime
+	}
+	st, err := os.Stat(exePath)
+	if err != nil {
+		return modTime
+	}
+	return st.ModTime()
+})
+
+func GenerateEmbedBindata(fsRootPath, outputFile string) error {
+	output, err := os.OpenFile(outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	defer output.Close()
+
+	meta := &EmbeddedMeta{}
+	meta.Root = &embeddedFileInfo{}
+	var outputOffset int64
+	var embedFiles func(parent *embeddedFileInfo, fsPath, embedPath string) error
+	embedFiles = func(parent *embeddedFileInfo, fsPath, embedPath string) error {
+		dirEntries, err := os.ReadDir(fsPath)
+		if err != nil {
+			return err
+		}
+		for _, dirEntry := range dirEntries {
+			if err != nil {
+				return err
+			}
+			if dirEntry.IsDir() {
+				child := &embeddedFileInfo{
+					BaseName: dirEntry.Name(),
+					Children: []*embeddedFileInfo{}, // non-nil means it's a directory
+				}
+				parent.Children = append(parent.Children, child)
+				if err = embedFiles(child, filepath.Join(fsPath, dirEntry.Name()), path.Join(embedPath, dirEntry.Name())); err != nil {
+					return err
+				}
+			} else {
+				data, err := os.ReadFile(filepath.Join(fsPath, dirEntry.Name()))
+				if err != nil {
+					return err
+				}
+				var compressed bytes.Buffer
+				gz, _ := gzip.NewWriterLevel(&compressed, gzip.BestCompression)
+				if _, err = gz.Write(data); err != nil {
+					return err
+				}
+				if err = gz.Close(); err != nil {
+					return err
+				}
+
+				// only use the compressed data if it is smaller than the original data
+				outputBytes := util.Iif(len(compressed.Bytes()) < len(data), compressed.Bytes(), data)
+				child := &embeddedFileInfo{
+					BaseName:   dirEntry.Name(),
+					OriginSize: int64(len(data)),
+					DataBegin:  outputOffset,
+					DataLen:    int64(len(outputBytes)),
+				}
+				if _, err = output.Write(outputBytes); err != nil {
+					return err
+				}
+				outputOffset += child.DataLen
+				parent.Children = append(parent.Children, child)
+			}
+		}
+		return nil
+	}
+
+	if err = embedFiles(meta.Root, fsRootPath, ""); err != nil {
+		return err
+	}
+	jsonBuf, err := json.Marshal(meta) // can't use json.NewEncoder here because it writes extra EOL
+	if err != nil {
+		return err
+	}
+	_, _ = output.Write([]byte{'\n'})
+	_, err = output.Write(jsonBuf)
+	return err
+}

modules/assetfs/embed_test.go

@@ -0,0 +1,98 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package assetfs
+
+import (
+	"bytes"
+	"io/fs"
+	"net/http"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEmbed(t *testing.T) {
+	tmpDir := t.TempDir()
+	tmpDataDir := tmpDir + "/data"
+	_ = os.MkdirAll(tmpDataDir+"/foo/bar", 0o755)
+	_ = os.WriteFile(tmpDataDir+"/a.txt", []byte("a"), 0o644)
+	_ = os.WriteFile(tmpDataDir+"/foo/bar/b.txt", bytes.Repeat([]byte("a"), 1000), 0o644)
+	_ = os.WriteFile(tmpDataDir+"/foo/c.txt", []byte("c"), 0o644)
+	require.NoError(t, GenerateEmbedBindata(tmpDataDir, tmpDir+"/out.dat"))
+
+	data, err := os.ReadFile(tmpDir + "/out.dat")
+	require.NoError(t, err)
+	efs := NewEmbeddedFS(data)
+
+	// test a non-existing file
+	_, err = fs.ReadFile(efs, "not exist")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	// test a normal file (no compression)
+	content, err := fs.ReadFile(efs, "a.txt")
+	require.NoError(t, err)
+	assert.Equal(t, "a", string(content))
+	fi, err := fs.Stat(efs, "a.txt")
+	require.NoError(t, err)
+	_, ok := fi.(EmbeddedFileInfo).GetGzipContent()
+	assert.False(t, ok)
+
+	// test a compressed file
+	content, err = fs.ReadFile(efs, "foo/bar/b.txt")
+	require.NoError(t, err)
+	assert.Equal(t, bytes.Repeat([]byte("a"), 1000), content)
+	fi, err = fs.Stat(efs, "foo/bar/b.txt")
+	require.NoError(t, err)
+	assert.False(t, fi.Mode().IsDir())
+	assert.True(t, fi.Mode().IsRegular())
+	gzipContent, ok := fi.(EmbeddedFileInfo).GetGzipContent()
+	assert.True(t, ok)
+	assert.Greater(t, len(gzipContent), 1)
+	assert.Less(t, len(gzipContent), 1000)
+
+	// test list root directory
+	entries, err := fs.ReadDir(efs, ".")
+	require.NoError(t, err)
+	assert.Len(t, entries, 2)
+	assert.Equal(t, "a.txt", entries[0].Name())
+	assert.False(t, entries[0].IsDir())
+
+	// test list subdirectory
+	entries, err = fs.ReadDir(efs, "foo")
+	require.NoError(t, err)
+	require.Len(t, entries, 2)
+	assert.Equal(t, "bar", entries[0].Name())
+	assert.True(t, entries[0].IsDir())
+	assert.Equal(t, "c.txt", entries[1].Name())
+	assert.False(t, entries[1].IsDir())
+
+	// test directory mode
+	fi, err = fs.Stat(efs, "foo")
+	require.NoError(t, err)
+	assert.True(t, fi.IsDir())
+	assert.True(t, fi.Mode().IsDir())
+	assert.False(t, fi.Mode().IsRegular())
+
+	// test httpfs
+	hfs := http.FS(efs)
+	hf, err := hfs.Open("foo/bar/b.txt")
+	require.NoError(t, err)
+	hi, err := hf.Stat()
+	require.NoError(t, err)
+	fiEmbedded, ok := hi.(EmbeddedFileInfo)
+	require.True(t, ok)
+	gzipContent, ok = fiEmbedded.GetGzipContent()
+	assert.True(t, ok)
+	assert.Greater(t, len(gzipContent), 1)
+	assert.Less(t, len(gzipContent), 1000)
+
+	// test httpfs directory listing
+	hf, err = hfs.Open("foo")
+	require.NoError(t, err)
+	dirs, err := hf.Readdir(1)
+	require.NoError(t, err)
+	assert.Len(t, dirs, 1)
+}

modules/assetfs/layered.go

@@ -0,0 +1,256 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package assetfs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"code.gitea.io/gitea/modules/container"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/process"
+	"code.gitea.io/gitea/modules/util"
+
+	"github.com/fsnotify/fsnotify"
+)
+
+// Layer represents a layer in a layered asset file-system. It has a name and works like http.FileSystem
+type Layer struct {
+	name      string
+	fs        http.FileSystem
+	localPath string
+}
+
+func (l *Layer) Name() string {
+	return l.name
+}
+
+// Open opens the named file. The caller is responsible for closing the file.
+func (l *Layer) Open(name string) (http.File, error) {
+	return l.fs.Open(name)
+}
+
+// Local returns a new Layer with the given name, it serves files from the given local path.
+func Local(name, base string, sub ...string) *Layer {
+	// TODO: the old behavior (StaticRootPath might not be absolute), not ideal, just keep the same as before
+	// Ideally, the caller should guarantee the base is absolute, guessing a relative path based on the current working directory is unreliable.
+	base, err := filepath.Abs(base)
+	if err != nil {
+		// This should never happen in a real system. If it happens, the user must have already been in trouble: the system is not able to resolve its own paths.
+		panic(fmt.Sprintf("Unable to get absolute path for %q: %v", base, err))
+	}
+	root := util.FilePathJoinAbs(base, sub...)
+	return &Layer{name: name, fs: http.Dir(root), localPath: root}
+}
+
+// Bindata returns a new Layer with the given name, it serves files from the given bindata asset.
+func Bindata(name string, fs fs.FS) *Layer {
+	return &Layer{name: name, fs: http.FS(fs)}
+}
+
+// LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers.
+// The first layer is the top layer, and it will be used first.
+// If the file is not found in the top layer, it will be searched in the next layer.
+type LayeredFS struct {
+	layers []*Layer
+}
+
+// Layered returns a new LayeredFS with the given layers. The first layer is the top layer.
+func Layered(layers ...*Layer) *LayeredFS {
+	return &LayeredFS{layers: layers}
+}
+
+// Open opens the named file. The caller is responsible for closing the file.
+func (l *LayeredFS) Open(name string) (http.File, error) {
+	for _, layer := range l.layers {
+		f, err := layer.Open(name)
+		if err == nil || !os.IsNotExist(err) {
+			return f, err
+		}
+	}
+	return nil, fs.ErrNotExist
+}
+
+// ReadFile reads the named file.
+func (l *LayeredFS) ReadFile(elems ...string) ([]byte, error) {
+	bs, _, err := l.ReadLayeredFile(elems...)
+	return bs, err
+}
+
+// ReadLayeredFile reads the named file, and returns the layer name.
+func (l *LayeredFS) ReadLayeredFile(elems ...string) ([]byte, string, error) {
+	name := util.PathJoinRel(elems...)
+	for _, layer := range l.layers {
+		f, err := layer.Open(name)
+		if os.IsNotExist(err) {
+			continue
+		} else if err != nil {
+			return nil, layer.name, err
+		}
+		bs, err := io.ReadAll(f)
+		_ = f.Close()
+		return bs, layer.name, err
+	}
+	return nil, "", fs.ErrNotExist
+}
+
+func shouldInclude(info fs.FileInfo, fileMode ...bool) bool {
+	if util.IsCommonHiddenFileName(info.Name()) {
+		return false
+	}
+	if len(fileMode) == 0 {
+		return true
+	} else if len(fileMode) == 1 {
+		return fileMode[0] == !info.Mode().IsDir()
+	}
+	panic("too many arguments for fileMode in shouldInclude")
+}
+
+func readDir(layer *Layer, name string) ([]fs.FileInfo, error) {
+	f, err := layer.Open(name)
+	if os.IsNotExist(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return f.Readdir(-1)
+}
+
+// ListFiles lists files/directories in the given directory. The fileMode controls the returned files.
+// * omitted: all files and directories will be returned.
+// * true: only files will be returned.
+// * false: only directories will be returned.
+// The returned files are sorted by name.
+func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) {
+	fileSet := make(container.Set[string])
+	for _, layer := range l.layers {
+		infos, err := readDir(layer, name)
+		if err != nil {
+			return nil, err
+		}
+		for _, info := range infos {
+			if shouldInclude(info, fileMode...) {
+				fileSet.Add(info.Name())
+			}
+		}
+	}
+	files := fileSet.Values()
+	sort.Strings(files)
+	return files, nil
+}
+
+// ListAllFiles returns files/directories in the given directory, including subdirectories, recursively.
+// The fileMode controls the returned files:
+// * omitted: all files and directories will be returned.
+// * true: only files will be returned.
+// * false: only directories will be returned.
+// The returned files are sorted by name.
+func (l *LayeredFS) ListAllFiles(name string, fileMode ...bool) ([]string, error) {
+	return listAllFiles(l.layers, name, fileMode...)
+}
+
+func listAllFiles(layers []*Layer, name string, fileMode ...bool) ([]string, error) {
+	fileSet := make(container.Set[string])
+	var list func(dir string) error
+	list = func(dir string) error {
+		for _, layer := range layers {
+			infos, err := readDir(layer, dir)
+			if err != nil {
+				return err
+			}
+			for _, info := range infos {
+				path := util.PathJoinRelX(dir, info.Name())
+				if shouldInclude(info, fileMode...) {
+					fileSet.Add(path)
+				}
+				if info.IsDir() {
+					if err = list(path); err != nil {
+						return err
+					}
+				}
+			}
+		}
+		return nil
+	}
+	if err := list(name); err != nil {
+		return nil, err
+	}
+	files := fileSet.Values()
+	sort.Strings(files)
+	return files, nil
+}
+
+// WatchLocalChanges watches local changes in the file-system. It's used to help to reload assets when the local file-system changes.
+func (l *LayeredFS) WatchLocalChanges(ctx context.Context, callback func()) {
+	ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Asset Local FileSystem Watcher", process.SystemProcessType, true)
+	defer finished()
+
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Error("Unable to create watcher for asset local file-system: %v", err)
+		return
+	}
+	defer watcher.Close()
+
+	for _, layer := range l.layers {
+		if layer.localPath == "" {
+			continue
+		}
+		layerDirs, err := listAllFiles([]*Layer{layer}, ".", false)
+		if err != nil {
+			log.Error("Unable to list directories for asset local file-system %q: %v", layer.localPath, err)
+			continue
+		}
+		layerDirs = append(layerDirs, ".")
+		for _, dir := range layerDirs {
+			if err = watcher.Add(util.FilePathJoinAbs(layer.localPath, dir)); err != nil && !os.IsNotExist(err) {
+				log.Error("Unable to watch directory %s: %v", dir, err)
+			}
+		}
+	}
+
+	debounce := util.Debounce(100 * time.Millisecond)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case event, ok := <-watcher.Events:
+			if !ok {
+				return
+			}
+			log.Trace("Watched asset local file-system had event: %v", event)
+			debounce(callback)
+		case err, ok := <-watcher.Errors:
+			if !ok {
+				return
+			}
+			log.Error("Watched asset local file-system had error: %v", err)
+		}
+	}
+}
+
+// GetFileLayerName returns the name of the first-seen layer that contains the given file.
+func (l *LayeredFS) GetFileLayerName(elems ...string) string {
+	name := util.PathJoinRel(elems...)
+	for _, layer := range l.layers {
+		f, err := layer.Open(name)
+		if os.IsNotExist(err) {
+			continue
+		} else if err != nil {
+			return ""
+		}
+		_ = f.Close()
+		return layer.name
+	}
+	return ""
+}

modules/assetfs/layered_test.go

@@ -0,0 +1,109 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package assetfs
+
+import (
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLayered(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "assetfs-layers")
+	dir1 := filepath.Join(dir, "l1")
+	dir2 := filepath.Join(dir, "l2")
+
+	mkdir := func(elems ...string) {
+		assert.NoError(t, os.MkdirAll(filepath.Join(elems...), 0o755))
+	}
+	write := func(content string, elems ...string) {
+		assert.NoError(t, os.WriteFile(filepath.Join(elems...), []byte(content), 0o644))
+	}
+
+	// d1 & f1: only in "l1"; d2 & f2: only in "l2"
+	// da & fa: in both "l1" and "l2"
+	mkdir(dir1, "d1")
+	mkdir(dir1, "da")
+	mkdir(dir1, "da/sub1")
+
+	mkdir(dir2, "d2")
+	mkdir(dir2, "da")
+	mkdir(dir2, "da/sub2")
+
+	write("dummy", dir1, ".DS_Store")
+	write("f1", dir1, "f1")
+	write("fa-1", dir1, "fa")
+	write("d1-f", dir1, "d1/f")
+	write("da-f-1", dir1, "da/f")
+
+	write("f2", dir2, "f2")
+	write("fa-2", dir2, "fa")
+	write("d2-f", dir2, "d2/f")
+	write("da-f-2", dir2, "da/f")
+
+	assets := Layered(Local("l1", dir1), Local("l2", dir2))
+
+	f, err := assets.Open("f1")
+	assert.NoError(t, err)
+	bs, err := io.ReadAll(f)
+	assert.NoError(t, err)
+	assert.Equal(t, "f1", string(bs))
+	_ = f.Close()
+
+	assertRead := func(expected string, expectedErr error, elems ...string) {
+		bs, err := assets.ReadFile(elems...)
+		if err != nil {
+			assert.ErrorIs(t, err, expectedErr)
+		} else {
+			assert.NoError(t, err)
+			assert.Equal(t, expected, string(bs))
+		}
+	}
+	assertRead("f1", nil, "f1")
+	assertRead("f2", nil, "f2")
+	assertRead("fa-1", nil, "fa")
+
+	assertRead("d1-f", nil, "d1/f")
+	assertRead("d2-f", nil, "d2/f")
+	assertRead("da-f-1", nil, "da/f")
+
+	assertRead("", fs.ErrNotExist, "no-such")
+
+	files, err := assets.ListFiles(".", true)
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"f1", "f2", "fa"}, files)
+
+	files, err = assets.ListFiles(".", false)
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"d1", "d2", "da"}, files)
+
+	files, err = assets.ListFiles(".")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files)
+
+	files, err = assets.ListAllFiles(".", true)
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files)
+
+	files, err = assets.ListAllFiles(".", false)
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files)
+
+	files, err = assets.ListAllFiles(".")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{
+		"d1", "d1/f",
+		"d2", "d2/f",
+		"da", "da/f", "da/sub1", "da/sub2",
+		"f1", "f2", "fa",
+	}, files)
+
+	assert.Empty(t, assets.GetFileLayerName("no-such"))
+	assert.Equal(t, "l1", assets.GetFileLayerName("f1"))
+	assert.Equal(t, "l2", assets.GetFileLayerName("f2"))
+}

modules/auth/common.go

@@ -0,0 +1,22 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package auth
+
+import (
+	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/log"
+)
+
+func UnmarshalGroupTeamMapping(raw string) (map[string]map[string][]string, error) {
+	groupTeamMapping := make(map[string]map[string][]string)
+	if raw == "" {
+		return groupTeamMapping, nil
+	}
+	err := json.Unmarshal([]byte(raw), &groupTeamMapping)
+	if err != nil {
+		log.Error("Failed to unmarshal group team mapping: %v", err)
+		return nil, err
+	}
+	return groupTeamMapping, nil
+}

modules/auth/httpauth/httpauth.go

@@ -0,0 +1,47 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package httpauth
+
+import (
+	"encoding/base64"
+	"strings"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+type BasicAuth struct {
+	Username, Password string
+}
+
+type BearerToken struct {
+	Token string
+}
+
+type ParsedAuthorizationHeader struct {
+	BasicAuth   *BasicAuth
+	BearerToken *BearerToken
+}
+
+func ParseAuthorizationHeader(header string) (ret ParsedAuthorizationHeader, _ bool) {
+	parts := strings.Fields(header)
+	if len(parts) != 2 {
+		return ret, false
+	}
+	if util.AsciiEqualFold(parts[0], "basic") {
+		s, err := base64.StdEncoding.DecodeString(parts[1])
+		if err != nil {
+			return ret, false
+		}
+		u, p, ok := strings.Cut(string(s), ":")
+		if !ok {
+			return ret, false
+		}
+		ret.BasicAuth = &BasicAuth{Username: u, Password: p}
+		return ret, true
+	} else if util.AsciiEqualFold(parts[0], "token") || util.AsciiEqualFold(parts[0], "bearer") {
+		ret.BearerToken = &BearerToken{Token: parts[1]}
+		return ret, true
+	}
+	return ret, false
+}

modules/auth/httpauth/httpauth_test.go

@@ -0,0 +1,43 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package httpauth
+
+import (
+	"encoding/base64"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseAuthorizationHeader(t *testing.T) {
+	type parsed = ParsedAuthorizationHeader
+	type basic = BasicAuth
+	type bearer = BearerToken
+	cases := []struct {
+		headerValue string
+		expected    parsed
+		ok          bool
+	}{
+		{"", parsed{}, false},
+		{"?", parsed{}, false},
+		{"foo", parsed{}, false},
+		{"any value", parsed{}, false},
+
+		{"Basic ?", parsed{}, false},
+		{"Basic " + base64.StdEncoding.EncodeToString([]byte("foo")), parsed{}, false},
+		{"Basic " + base64.StdEncoding.EncodeToString([]byte("foo:bar")), parsed{BasicAuth: &basic{"foo", "bar"}}, true},
+		{"basic " + base64.StdEncoding.EncodeToString([]byte("foo:bar")), parsed{BasicAuth: &basic{"foo", "bar"}}, true},
+
+		{"token value", parsed{BearerToken: &bearer{"value"}}, true},
+		{"Token value", parsed{BearerToken: &bearer{"value"}}, true},
+		{"bearer value", parsed{BearerToken: &bearer{"value"}}, true},
+		{"Bearer value", parsed{BearerToken: &bearer{"value"}}, true},
+		{"Bearer wrong value", parsed{}, false},
+	}
+	for _, c := range cases {
+		ret, ok := ParseAuthorizationHeader(c.headerValue)
+		assert.Equal(t, c.ok, ok, "header %q", c.headerValue)
+		assert.Equal(t, c.expected, ret, "header %q", c.headerValue)
+	}
+}

modules/auth/openid/discovery_cache.go

@@ -0,0 +1,57 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package openid
+
+import (
+	"sync"
+	"time"
+
+	"github.com/yohcop/openid-go"
+)
+
+type timedDiscoveredInfo struct {
+	info openid.DiscoveredInfo
+	time time.Time
+}
+
+type timedDiscoveryCache struct {
+	cache map[string]timedDiscoveredInfo
+	ttl   time.Duration
+	mutex *sync.Mutex
+}
+
+func newTimedDiscoveryCache(ttl time.Duration) *timedDiscoveryCache {
+	return &timedDiscoveryCache{cache: map[string]timedDiscoveredInfo{}, ttl: ttl, mutex: &sync.Mutex{}}
+}
+
+func (s *timedDiscoveryCache) Put(id string, info openid.DiscoveredInfo) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	s.cache[id] = timedDiscoveredInfo{info: info, time: time.Now()}
+}
+
+// Delete timed-out cache entries
+func (s *timedDiscoveryCache) cleanTimedOut() {
+	now := time.Now()
+	for k, e := range s.cache {
+		diff := now.Sub(e.time)
+		if diff > s.ttl {
+			delete(s.cache, k)
+		}
+	}
+}
+
+func (s *timedDiscoveryCache) Get(id string) openid.DiscoveredInfo {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	// Delete old cached while we are at it.
+	s.cleanTimedOut()
+
+	if info, has := s.cache[id]; has {
+		return info.info
+	}
+	return nil
+}

modules/auth/openid/discovery_cache_test.go

@@ -0,0 +1,49 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package openid
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type testDiscoveredInfo struct{}
+
+func (s *testDiscoveredInfo) ClaimedID() string {
+	return "claimedID"
+}
+
+func (s *testDiscoveredInfo) OpEndpoint() string {
+	return "opEndpoint"
+}
+
+func (s *testDiscoveredInfo) OpLocalID() string {
+	return "opLocalID"
+}
+
+func TestTimedDiscoveryCache(t *testing.T) {
+	ttl := 50 * time.Millisecond
+	dc := newTimedDiscoveryCache(ttl)
+
+	// Put some initial values
+	dc.Put("foo", &testDiscoveredInfo{}) // openid.opEndpoint: "a", openid.opLocalID: "b", openid.claimedID: "c"})
+
+	// Make sure we can retrieve them
+	di := dc.Get("foo")
+	require.NotNil(t, di)
+	assert.Equal(t, "opEndpoint", di.OpEndpoint())
+	assert.Equal(t, "opLocalID", di.OpLocalID())
+	assert.Equal(t, "claimedID", di.ClaimedID())
+
+	// Attempt to get a non-existent value
+	assert.Nil(t, dc.Get("bar"))
+
+	// Sleep for a while and try to retrieve again
+	time.Sleep(ttl * 3 / 2)
+
+	assert.Nil(t, dc.Get("foo"))
+}

modules/auth/openid/openid.go

@@ -0,0 +1,37 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package openid
+
+import (
+	"time"
+
+	"github.com/yohcop/openid-go"
+)
+
+// For the demo, we use in-memory infinite storage nonce and discovery
+// cache. In your app, do not use this as it will eat up memory and
+// never
+// free it. Use your own implementation, on a better database system.
+// If you have multiple servers for example, you may need to share at
+// least
+// the nonceStore between them.
+var (
+	nonceStore     = openid.NewSimpleNonceStore()
+	discoveryCache = newTimedDiscoveryCache(24 * time.Hour)
+)
+
+// Verify handles response from OpenID provider
+func Verify(fullURL string) (id string, err error) {
+	return openid.Verify(fullURL, discoveryCache, nonceStore)
+}
+
+// Normalize normalizes an OpenID URI
+func Normalize(url string) (id string, err error) {
+	return openid.Normalize(url)
+}
+
+// RedirectURL redirects browser
+func RedirectURL(id, callbackURL, realm string) (string, error) {
+	return openid.RedirectURL(id, callbackURL, realm)
+}

modules/auth/pam/pam.go

@@ -0,0 +1,43 @@
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build pam
+
+package pam
+
+import (
+	"errors"
+
+	"github.com/msteinert/pam"
+)
+
+// Supported is true when built with PAM
+var Supported = true
+
+// Auth pam auth service
+func Auth(serviceName, userName, passwd string) (string, error) {
+	t, err := pam.StartFunc(serviceName, userName, func(s pam.Style, msg string) (string, error) {
+		switch s {
+		case pam.PromptEchoOff:
+			return passwd, nil
+		case pam.PromptEchoOn, pam.ErrorMsg, pam.TextInfo:
+			return "", nil
+		}
+		return "", errors.New("Unrecognized PAM message style")
+	})
+	if err != nil {
+		return "", err
+	}
+
+	if err = t.Authenticate(0); err != nil {
+		return "", err
+	}
+
+	if err = t.AcctMgmt(0); err != nil {
+		return "", err
+	}
+
+	// PAM login names might suffer transformations in the PAM stack.
+	// We should take whatever the PAM stack returns for it.
+	return t.GetItem(pam.User)
+}

modules/auth/pam/pam_stub.go

@@ -0,0 +1,22 @@
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build !pam
+
+package pam
+
+import (
+	"errors"
+)
+
+// Supported is false when built without PAM
+var Supported = false
+
+// Auth not supported lack of pam tag
+func Auth(serviceName, userName, passwd string) (string, error) {
+	// bypass the lint on callers: SA4023: this comparison is always true (staticcheck)
+	if !Supported {
+		return "", errors.New("PAM not supported")
+	}
+	return "", nil
+}

modules/auth/pam/pam_test.go

@@ -0,0 +1,19 @@
+//go:build pam
+
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package pam
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPamAuth(t *testing.T) {
+	result, err := Auth("gitea", "user1", "false-pwd")
+	assert.Error(t, err)
+	assert.EqualError(t, err, "Authentication failure")
+	assert.Empty(t, result)
+}

modules/auth/password/hash/argon2.go

@@ -0,0 +1,80 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package hash
+
+import (
+	"encoding/hex"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+
+	"golang.org/x/crypto/argon2"
+)
+
+func init() {
+	MustRegister("argon2", NewArgon2Hasher)
+}
+
+// Argon2Hasher implements PasswordHasher
+// and uses the Argon2 key derivation function, hybrant variant
+type Argon2Hasher struct {
+	time    uint32
+	memory  uint32
+	threads uint8
+	keyLen  uint32
+}
+
+// HashWithSaltBytes a provided password and salt
+func (hasher *Argon2Hasher) HashWithSaltBytes(password string, salt []byte) string {
+	if hasher == nil {
+		return ""
+	}
+	return hex.EncodeToString(argon2.IDKey([]byte(password), salt, hasher.time, hasher.memory, hasher.threads, hasher.keyLen))
+}
+
+// NewArgon2Hasher is a factory method to create an Argon2Hasher
+// The provided config should be either empty or of the form:
+// "<time>$<memory>$<threads>$<keyLen>", where <x> is the string representation
+// of an integer
+func NewArgon2Hasher(config string) *Argon2Hasher {
+	// This default configuration uses the following parameters:
+	// time=2, memory=64*1024, threads=8, keyLen=50.
+	// It will make two passes through the memory, using 64MiB in total.
+	// This matches the original configuration for `argon2` prior to storing hash parameters
+	// in the database.
+	// THESE VALUES MUST NOT BE CHANGED OR BACKWARDS COMPATIBILITY WILL BREAK
+	hasher := &Argon2Hasher{
+		time:    2,
+		memory:  1 << 16,
+		threads: 8,
+		keyLen:  50,
+	}
+
+	if config == "" {
+		return hasher
+	}
+
+	vals := strings.SplitN(config, "$", 4)
+	if len(vals) != 4 {
+		log.Error("invalid argon2 hash spec %s", config)
+		return nil
+	}
+
+	parsed, err := parseUIntParam(vals[0], "time", "argon2", config, nil)
+	hasher.time = uint32(parsed)
+
+	parsed, err = parseUIntParam(vals[1], "memory", "argon2", config, err)
+	hasher.memory = uint32(parsed)
+
+	parsed, err = parseUIntParam(vals[2], "threads", "argon2", config, err)
+	hasher.threads = uint8(parsed)
+
+	parsed, err = parseUIntParam(vals[3], "keyLen", "argon2", config, err)
+	hasher.keyLen = uint32(parsed)
+	if err != nil {
+		return nil
+	}
+
+	return hasher
+}

modules/auth/password/hash/bcrypt.go

@@ -0,0 +1,54 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package hash
+
+import (
+	"golang.org/x/crypto/bcrypt"
+)
+
+func init() {
+	MustRegister("bcrypt", NewBcryptHasher)
+}
+
+// BcryptHasher implements PasswordHasher
+// and uses the bcrypt password hash function.
+type BcryptHasher struct {
+	cost int
+}
+
+// HashWithSaltBytes a provided password and salt
+func (hasher *BcryptHasher) HashWithSaltBytes(password string, salt []byte) string {
+	if hasher == nil {
+		return ""
+	}
+	hashedPassword, _ := bcrypt.GenerateFromPassword([]byte(password), hasher.cost)
+	return string(hashedPassword)
+}
+
+func (hasher *BcryptHasher) VerifyPassword(password, hashedPassword, salt string) bool {
+	return bcrypt.CompareHashAndPassword([]byte(hashedPassword), []byte(password)) == nil
+}
+
+// NewBcryptHasher is a factory method to create an BcryptHasher
+// The provided config should be either empty or the string representation of the "<cost>"
+// as an integer
+func NewBcryptHasher(config string) *BcryptHasher {
+	// This matches the original configuration for `bcrypt` prior to storing hash parameters
+	// in the database.
+	// THESE VALUES MUST NOT BE CHANGED OR BACKWARDS COMPATIBILITY WILL BREAK
+	hasher := &BcryptHasher{
+		cost: 10, // cost=10. i.e. 2^10 rounds of key expansion.
+	}
+
+	if config == "" {
+		return hasher
+	}
+	var err error
+	hasher.cost, err = parseIntParam(config, "cost", "bcrypt", config, nil)
+	if err != nil {
+		return nil
+	}
+
+	return hasher
+}

modules/auth/password/hash/common.go

@@ -0,0 +1,28 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package hash
+
+import (
+	"strconv"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+func parseIntParam(value, param, algorithmName, config string, previousErr error) (int, error) {
+	parsed, err := strconv.Atoi(value)
+	if err != nil {
+		log.Error("invalid integer for %s representation in %s hash spec %s", param, algorithmName, config)
+		return 0, err
+	}
+	return parsed, previousErr // <- Keep the previous error as this function should still return an error once everything has been checked if any call failed
+}
+
+func parseUIntParam(value, param, algorithmName, config string, previousErr error) (uint64, error) { //nolint:unparam // algorithmName is always argon2
+	parsed, err := strconv.ParseUint(value, 10, 64)
+	if err != nil {
+		log.Error("invalid integer for %s representation in %s hash spec %s", param, algorithmName, config)
+		return 0, err
+	}
+	return parsed, previousErr // <- Keep the previous error as this function should still return an error once everything has been checked if any call failed
+}

modules/auth/password/hash/dummy.go

@@ -0,0 +1,33 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package hash
+
+import (
+	"encoding/hex"
+)
+
+// DummyHasher implements PasswordHasher and is a dummy hasher that simply
+// puts the password in place with its salt
+// This SHOULD NOT be used in production and is provided to make the integration
+// tests faster only
+type DummyHasher struct{}
+
+// HashWithSaltBytes a provided password and salt
+func (hasher *DummyHasher) HashWithSaltBytes(password string, salt []byte) string {
+	if hasher == nil {
+		return ""
+	}
+
+	if len(salt) == 10 {
+		return string(salt) + ":" + password
+	}
+
+	return hex.EncodeToString(salt) + ":" + password
+}
+
+// NewDummyHasher is a factory method to create a DummyHasher
+// Any provided configuration is ignored
+func NewDummyHasher(_ string) *DummyHasher {
+	return &DummyHasher{}
+}

modules/auth/password/hash/dummy_test.go

@@ -0,0 +1,25 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package hash
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDummyHasher(t *testing.T) {
+	dummy := &PasswordHashAlgorithm{
+		PasswordSaltHasher: NewDummyHasher(""),
+		Specification:      "dummy",
+	}
+
+	password, salt := "password", "ZogKvWdyEx"
+
+	hash, err := dummy.Hash(password, salt)
+	assert.NoError(t, err)
+	assert.Equal(t, hash, salt+":"+password)
+
+	assert.True(t, dummy.VerifyPassword(password, hash, salt))
+}

modules/auth/password/hash/hash.go

@@ -0,0 +1,189 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package hash
+
+import (
+	"crypto/subtle"
+	"encoding/hex"
+	"fmt"
+	"strings"
+	"sync/atomic"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+// This package takes care of hashing passwords, verifying passwords, defining
+// available password algorithms, defining recommended password algorithms and
+// choosing the default password algorithm.
+
+// PasswordSaltHasher will hash a provided password with the provided saltBytes
+type PasswordSaltHasher interface {
+	HashWithSaltBytes(password string, saltBytes []byte) string
+}
+
+// PasswordHasher will hash a provided password with the salt
+type PasswordHasher interface {
+	Hash(password, salt string) (string, error)
+}
+
+// PasswordVerifier will ensure that a providedPassword matches the hashPassword when hashed with the salt
+type PasswordVerifier interface {
+	VerifyPassword(providedPassword, hashedPassword, salt string) bool
+}
+
+// PasswordHashAlgorithms are named PasswordSaltHashers with a default verifier and hash function
+type PasswordHashAlgorithm struct {
+	PasswordSaltHasher
+	Specification string // The specification that is used to create the internal PasswordSaltHasher
+}
+
+// Hash the provided password with the salt and return the hash
+func (algorithm *PasswordHashAlgorithm) Hash(password, salt string) (string, error) {
+	var saltBytes []byte
+
+	// There are two formats for the salt value:
+	// * The new format is a (32+)-byte hex-encoded string
+	// * The old format was a 10-byte binary format
+	// We have to tolerate both here.
+	if len(salt) == 10 {
+		saltBytes = []byte(salt)
+	} else {
+		var err error
+		saltBytes, err = hex.DecodeString(salt)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return algorithm.HashWithSaltBytes(password, saltBytes), nil
+}
+
+// Verify the provided password matches the hashPassword when hashed with the salt
+func (algorithm *PasswordHashAlgorithm) VerifyPassword(providedPassword, hashedPassword, salt string) bool {
+	// Some PasswordSaltHashers have their own specialised compare function that takes into
+	// account the stored parameters within the hash. e.g. bcrypt
+	if verifier, ok := algorithm.PasswordSaltHasher.(PasswordVerifier); ok {
+		return verifier.VerifyPassword(providedPassword, hashedPassword, salt)
+	}
+
+	// Compute the hash of the password.
+	providedPasswordHash, err := algorithm.Hash(providedPassword, salt)
+	if err != nil {
+		log.Error("passwordhash: %v.Hash(): %v", algorithm.Specification, err)
+		return false
+	}
+
+	// Compare it against the hashed password in constant-time.
+	return subtle.ConstantTimeCompare([]byte(hashedPassword), []byte(providedPasswordHash)) == 1
+}
+
+var (
+	lastNonDefaultAlgorithm  atomic.Value
+	availableHasherFactories = map[string]func(string) PasswordSaltHasher{}
+)
+
+// MustRegister registers a PasswordSaltHasher with the availableHasherFactories
+// Caution: This is not thread safe.
+func MustRegister[T PasswordSaltHasher](name string, newFn func(config string) T) {
+	if err := Register(name, newFn); err != nil {
+		panic(err)
+	}
+}
+
+// Register registers a PasswordSaltHasher with the availableHasherFactories
+// Caution: This is not thread safe.
+func Register[T PasswordSaltHasher](name string, newFn func(config string) T) error {
+	if _, has := availableHasherFactories[name]; has {
+		return fmt.Errorf("duplicate registration of password salt hasher: %s", name)
+	}
+
+	availableHasherFactories[name] = func(config string) PasswordSaltHasher {
+		n := newFn(config)
+		return n
+	}
+	return nil
+}
+
+// In early versions of gitea the password hash algorithm field of a user could be
+// empty. At that point the default was `pbkdf2` without configuration values
+//
+// Please note this is not the same as the DefaultAlgorithm which is used
+// to determine what an empty PASSWORD_HASH_ALGO setting in the app.ini means.
+// These are not the same even if they have the same apparent value and they mean different things.
+//
+// DO NOT COALESCE THESE VALUES
+const defaultEmptyHashAlgorithmSpecification = "pbkdf2"
+
+// Parse will convert the provided algorithm specification in to a PasswordHashAlgorithm
+// If the provided specification matches the DefaultHashAlgorithm Specification it will be
+// used.
+// In addition the last non-default hasher will be cached to help reduce the load from
+// parsing specifications.
+//
+// NOTE: No de-aliasing is done in this function, thus any specification which does not
+// contain a configuration will use the default values for that hasher. These are not
+// necessarily the same values as those obtained by dealiasing. This allows for
+// seamless backwards compatibility with the original configuration.
+//
+// To further labour this point, running `Parse("pbkdf2")` does not obtain the
+// same algorithm as setting `PASSWORD_HASH_ALGO=pbkdf2` in app.ini, nor is it intended to.
+// A user that has `password_hash_algo='pbkdf2'` in the db means get the original, unconfigured algorithm
+// Users will be migrated automatically as they log-in to have the complete specification stored
+// in their `password_hash_algo` fields by other code.
+func Parse(algorithmSpec string) *PasswordHashAlgorithm {
+	if algorithmSpec == "" {
+		algorithmSpec = defaultEmptyHashAlgorithmSpecification
+	}
+
+	if DefaultHashAlgorithm != nil && algorithmSpec == DefaultHashAlgorithm.Specification {
+		return DefaultHashAlgorithm
+	}
+
+	ptr := lastNonDefaultAlgorithm.Load()
+	if ptr != nil {
+		hashAlgorithm, ok := ptr.(*PasswordHashAlgorithm)
+		if ok && hashAlgorithm.Specification == algorithmSpec {
+			return hashAlgorithm
+		}
+	}
+
+	// Now convert the provided specification in to a hasherType +/- some configuration parameters
+	vals := strings.SplitN(algorithmSpec, "$", 2)
+	var hasherType string
+	var config string
+
+	if len(vals) == 0 {
+		// This should not happen as algorithmSpec should not be empty
+		// due to it being assigned to defaultEmptyHashAlgorithmSpecification above
+		// but we should be absolutely cautious here
+		return nil
+	}
+
+	hasherType = vals[0]
+	if len(vals) > 1 {
+		config = vals[1]
+	}
+
+	newFn, has := availableHasherFactories[hasherType]
+	if !has {
+		// unknown hasher type
+		return nil
+	}
+
+	ph := newFn(config)
+	if ph == nil {
+		// The provided configuration is likely invalid - it will have been logged already
+		// but we cannot hash safely
+		return nil
+	}
+
+	hashAlgorithm := &PasswordHashAlgorithm{
+		PasswordSaltHasher: ph,
+		Specification:      algorithmSpec,
+	}
+
+	lastNonDefaultAlgorithm.Store(hashAlgorithm)
+
+	return hashAlgorithm
+}

modules/auth/password/hash/hash_test.go

@@ -0,0 +1,190 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package hash
+
+import (
+	"encoding/hex"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type testSaltHasher string
+
+func (t testSaltHasher) HashWithSaltBytes(password string, salt []byte) string {
+	return password + "$" + string(salt) + "$" + string(t)
+}
+
+func Test_registerHasher(t *testing.T) {
+	MustRegister("Test_registerHasher", func(config string) testSaltHasher {
+		return testSaltHasher(config)
+	})
+
+	assert.Panics(t, func() {
+		MustRegister("Test_registerHasher", func(config string) testSaltHasher {
+			return testSaltHasher(config)
+		})
+	})
+
+	assert.Error(t, Register("Test_registerHasher", func(config string) testSaltHasher {
+		return testSaltHasher(config)
+	}))
+
+	assert.Equal(t, "password$salt$",
+		Parse("Test_registerHasher").PasswordSaltHasher.HashWithSaltBytes("password", []byte("salt")))
+
+	assert.Equal(t, "password$salt$config",
+		Parse("Test_registerHasher$config").PasswordSaltHasher.HashWithSaltBytes("password", []byte("salt")))
+
+	delete(availableHasherFactories, "Test_registerHasher")
+}
+
+func TestParse(t *testing.T) {
+	hashAlgorithmsToTest := []string{}
+	for plainHashAlgorithmNames := range availableHasherFactories {
+		hashAlgorithmsToTest = append(hashAlgorithmsToTest, plainHashAlgorithmNames)
+	}
+	for _, aliased := range aliasAlgorithmNames {
+		if strings.Contains(aliased, "$") {
+			hashAlgorithmsToTest = append(hashAlgorithmsToTest, aliased)
+		}
+	}
+	for _, algorithmName := range hashAlgorithmsToTest {
+		t.Run(algorithmName, func(t *testing.T) {
+			algo := Parse(algorithmName)
+			assert.NotNil(t, algo, "Algorithm %s resulted in an empty algorithm", algorithmName)
+		})
+	}
+}
+
+func TestHashing(t *testing.T) {
+	hashAlgorithmsToTest := []string{}
+	for plainHashAlgorithmNames := range availableHasherFactories {
+		hashAlgorithmsToTest = append(hashAlgorithmsToTest, plainHashAlgorithmNames)
+	}
+	for _, aliased := range aliasAlgorithmNames {
+		if strings.Contains(aliased, "$") {
+			hashAlgorithmsToTest = append(hashAlgorithmsToTest, aliased)
+		}
+	}
+
+	runTests := func(password, salt string, shouldPass bool) {
+		for _, algorithmName := range hashAlgorithmsToTest {
+			t.Run(algorithmName, func(t *testing.T) {
+				output, err := Parse(algorithmName).Hash(password, salt)
+				if shouldPass {
+					assert.NoError(t, err)
+					assert.NotEmpty(t, output, "output for %s was empty", algorithmName)
+				} else {
+					assert.Error(t, err)
+				}
+
+				assert.Equal(t, Parse(algorithmName).VerifyPassword(password, output, salt), shouldPass)
+			})
+		}
+	}
+
+	// Test with new salt format.
+	runTests(strings.Repeat("a", 16), hex.EncodeToString([]byte{0x01, 0x02, 0x03}), true)
+
+	// Test with legacy salt format.
+	runTests(strings.Repeat("a", 16), strings.Repeat("b", 10), true)
+
+	// Test with invalid salt.
+	runTests(strings.Repeat("a", 16), "a", false)
+}
+
+// vectors were generated using the current codebase.
+var vectors = []struct {
+	algorithms []string
+	password   string
+	salt       string
+	output     string
+	shouldfail bool
+}{
+	{
+		algorithms: []string{"bcrypt", "bcrypt$10"},
+		password:   "abcdef",
+		salt:       strings.Repeat("a", 10),
+		output:     "$2a$10$fjtm8BsQ2crym01/piJroenO3oSVUBhSLKaGdTYJ4tG0ePVCrU0G2",
+		shouldfail: false,
+	},
+	{
+		algorithms: []string{"scrypt", "scrypt$65536$16$2$50"},
+		password:   "abcdef",
+		salt:       strings.Repeat("a", 10),
+		output:     "3b571d0c07c62d42b7bad3dbf18fb0cd67d4d8cd4ad4c6928e1090e5b2a4a84437c6fd2627d897c0e7e65025ca62b67a0002",
+		shouldfail: false,
+	},
+	{
+		algorithms: []string{"argon2", "argon2$2$65536$8$50"},
+		password:   "abcdef",
+		salt:       strings.Repeat("a", 10),
+		output:     "551f089f570f989975b6f7c6a8ff3cf89bc486dd7bbe87ed4d80ad4362f8ee599ec8dda78dac196301b98456402bcda775dc",
+		shouldfail: false,
+	},
+	{
+		algorithms: []string{"pbkdf2", "pbkdf2$10000$50"},
+		password:   "abcdef",
+		salt:       strings.Repeat("a", 10),
+		output:     "ab48d5471b7e6ed42d10001db88c852ff7303c788e49da5c3c7b63d5adf96360303724b74b679223a3dea8a242d10abb1913",
+		shouldfail: false,
+	},
+	{
+		algorithms: []string{"bcrypt", "bcrypt$10"},
+		password:   "abcdef",
+		salt:       hex.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04}),
+		output:     "$2a$10$qhgm32w9ZpqLygugWJsLjey8xRGcaq9iXAfmCeNBXxddgyoaOC3Gq",
+		shouldfail: false,
+	},
+	{
+		algorithms: []string{"scrypt", "scrypt$65536$16$2$50"},
+		password:   "abcdef",
+		salt:       hex.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04}),
+		output:     "25fe5f66b43fa4eb7b6717905317cd2223cf841092dc8e0a1e8c75720ad4846cb5d9387303e14bc3c69faa3b1c51ef4b7de1",
+		shouldfail: false,
+	},
+	{
+		algorithms: []string{"argon2", "argon2$2$65536$8$50"},
+		password:   "abcdef",
+		salt:       hex.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04}),
+		output:     "9c287db63a91d18bb1414b703216da4fc431387c1ae7c8acdb280222f11f0929831055dbfd5126a3b48566692e83ec750d2a",
+		shouldfail: false,
+	},
+	{
+		algorithms: []string{"pbkdf2", "pbkdf2$10000$50"},
+		password:   "abcdef",
+		salt:       hex.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04}),
+		output:     "45d6cdc843d65cf0eda7b90ab41435762a282f7df013477a1c5b212ba81dbdca2edf1ecc4b5cb05956bb9e0c37ab29315d78",
+		shouldfail: false,
+	},
+	{
+		algorithms: []string{"pbkdf2$320000$50"},
+		password:   "abcdef",
+		salt:       hex.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04}),
+		output:     "84e233114499e8721da80e85568e5b7b5900b3e49a30845fcda9d1e1756da4547d70f8740ac2b4a5d82f88cebcd27f21bfe2",
+		shouldfail: false,
+	},
+	{
+		algorithms: []string{"pbkdf2", "pbkdf2$10000$50"},
+		password:   "abcdef",
+		salt:       "",
+		output:     "",
+		shouldfail: true,
+	},
+}
+
+// Ensure that the current code will correctly verify against the test vectors.
+func TestVectors(t *testing.T) {
+	for i, vector := range vectors {
+		for _, algorithm := range vector.algorithms {
+			t.Run(strconv.Itoa(i)+": "+algorithm, func(t *testing.T) {
+				pa := Parse(algorithm)
+				assert.Equal(t, !vector.shouldfail, pa.VerifyPassword(vector.password, vector.output, vector.salt))
+			})
+		}
+	}
+}

modules/auth/password/hash/pbkdf2.go

@@ -0,0 +1,67 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package hash
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+
+	"golang.org/x/crypto/pbkdf2"
+)
+
+func init() {
+	MustRegister("pbkdf2", NewPBKDF2Hasher)
+}
+
+// PBKDF2Hasher implements PasswordHasher
+// and uses the PBKDF2 key derivation function.
+type PBKDF2Hasher struct {
+	iter, keyLen int
+}
+
+// HashWithSaltBytes a provided password and salt
+func (hasher *PBKDF2Hasher) HashWithSaltBytes(password string, salt []byte) string {
+	if hasher == nil {
+		return ""
+	}
+	return hex.EncodeToString(pbkdf2.Key([]byte(password), salt, hasher.iter, hasher.keyLen, sha256.New))
+}
+
+// NewPBKDF2Hasher is a factory method to create an PBKDF2Hasher
+// config should be either empty or of the form:
+// "<iter>$<keyLen>", where <x> is the string representation
+// of an integer
+func NewPBKDF2Hasher(config string) *PBKDF2Hasher {
+	// This default configuration uses the following parameters:
+	// iter=10000, keyLen=50.
+	// This matches the original configuration for `pbkdf2` prior to storing parameters
+	// in the database.
+	// THESE VALUES MUST NOT BE CHANGED OR BACKWARDS COMPATIBILITY WILL BREAK
+	hasher := &PBKDF2Hasher{
+		iter:   10_000,
+		keyLen: 50,
+	}
+
+	if config == "" {
+		return hasher
+	}
+
+	vals := strings.SplitN(config, "$", 2)
+	if len(vals) != 2 {
+		log.Error("invalid pbkdf2 hash spec %s", config)
+		return nil
+	}
+
+	var err error
+	hasher.iter, err = parseIntParam(vals[0], "iter", "pbkdf2", config, nil)
+	hasher.keyLen, err = parseIntParam(vals[1], "keyLen", "pbkdf2", config, err)
+	if err != nil {
+		return nil
+	}
+
+	return hasher
+}

modules/auth/password/hash/scrypt.go

@@ -0,0 +1,67 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package hash
+
+import (
+	"encoding/hex"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+
+	"golang.org/x/crypto/scrypt"
+)
+
+func init() {
+	MustRegister("scrypt", NewScryptHasher)
+}
+
+// ScryptHasher implements PasswordHasher
+// and uses the scrypt key derivation function.
+type ScryptHasher struct {
+	n, r, p, keyLen int
+}
+
+// HashWithSaltBytes a provided password and salt
+func (hasher *ScryptHasher) HashWithSaltBytes(password string, salt []byte) string {
+	if hasher == nil {
+		return ""
+	}
+	hashedPassword, _ := scrypt.Key([]byte(password), salt, hasher.n, hasher.r, hasher.p, hasher.keyLen)
+	return hex.EncodeToString(hashedPassword)
+}
+
+// NewScryptHasher is a factory method to create an ScryptHasher
+// The provided config should be either empty or of the form:
+// "<n>$<r>$<p>$<keyLen>", where <x> is the string representation
+// of an integer
+func NewScryptHasher(config string) *ScryptHasher {
+	// This matches the original configuration for `scrypt` prior to storing hash parameters
+	// in the database.
+	// THESE VALUES MUST NOT BE CHANGED OR BACKWARDS COMPATIBILITY WILL BREAK
+	hasher := &ScryptHasher{
+		n:      1 << 16,
+		r:      16,
+		p:      2, // 2 passes through memory - this default config will use 128MiB in total.
+		keyLen: 50,
+	}
+
+	if config == "" {
+		return hasher
+	}
+
+	vals := strings.SplitN(config, "$", 4)
+	if len(vals) != 4 {
+		log.Error("invalid scrypt hash spec %s", config)
+		return nil
+	}
+	var err error
+	hasher.n, err = parseIntParam(vals[0], "n", "scrypt", config, nil)
+	hasher.r, err = parseIntParam(vals[1], "r", "scrypt", config, err)
+	hasher.p, err = parseIntParam(vals[2], "p", "scrypt", config, err)
+	hasher.keyLen, err = parseIntParam(vals[3], "keyLen", "scrypt", config, err)
+	if err != nil {
+		return nil
+	}
+	return hasher
+}

modules/auth/password/hash/setting.go

@@ -0,0 +1,76 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package hash
+
+// DefaultHashAlgorithmName represents the default value of PASSWORD_HASH_ALGO
+// configured in app.ini.
+//
+// It is NOT the same and does NOT map to the defaultEmptyHashAlgorithmSpecification.
+//
+// It will be dealiased as per aliasAlgorithmNames whereas
+// defaultEmptyHashAlgorithmSpecification does not undergo dealiasing.
+const DefaultHashAlgorithmName = "pbkdf2"
+
+var DefaultHashAlgorithm *PasswordHashAlgorithm
+
+// aliasAlgorithNames provides a mapping between the value of PASSWORD_HASH_ALGO
+// configured in the app.ini and the parameters used within the hashers internally.
+//
+// If it is necessary to change the default parameters for any hasher in future you
+// should change these values and not those in argon2.go etc.
+var aliasAlgorithmNames = map[string]string{
+	"argon2":    "argon2$2$65536$8$50",
+	"bcrypt":    "bcrypt$10",
+	"scrypt":    "scrypt$65536$16$2$50",
+	"pbkdf2":    "pbkdf2_v2", // pbkdf2 should default to pbkdf2_v2
+	"pbkdf2_v1": "pbkdf2$10000$50",
+	// The latest PBKDF2 password algorithm is used as the default since it doesn't
+	// use a lot of  memory and is safer to use on less powerful devices.
+	"pbkdf2_v2": "pbkdf2$50000$50",
+	// The pbkdf2_hi password algorithm is offered as a stronger alternative to the
+	// slightly improved pbkdf2_v2 algorithm
+	"pbkdf2_hi": "pbkdf2$320000$50",
+}
+
+var RecommendedHashAlgorithms = []string{
+	"pbkdf2",
+	"argon2",
+	"bcrypt",
+	"scrypt",
+	"pbkdf2_hi",
+}
+
+// hashAlgorithmToSpec converts an algorithm name or a specification to a full algorithm specification
+func hashAlgorithmToSpec(algorithmName string) string {
+	if algorithmName == "" {
+		algorithmName = DefaultHashAlgorithmName
+	}
+	alias, has := aliasAlgorithmNames[algorithmName]
+	for has {
+		algorithmName = alias
+		alias, has = aliasAlgorithmNames[algorithmName]
+	}
+	return algorithmName
+}
+
+// SetDefaultPasswordHashAlgorithm will take a provided algorithmName and de-alias it to
+// a complete algorithm specification.
+func SetDefaultPasswordHashAlgorithm(algorithmName string) (string, *PasswordHashAlgorithm) {
+	algoSpec := hashAlgorithmToSpec(algorithmName)
+	// now we get a full specification, e.g. pbkdf2$50000$50 rather than pbdkf2
+	DefaultHashAlgorithm = Parse(algoSpec)
+	return algoSpec, DefaultHashAlgorithm
+}
+
+// ConfigHashAlgorithm will try to find a "recommended algorithm name" defined by RecommendedHashAlgorithms for config
+// This function is not fast and is only used for the installation page
+func ConfigHashAlgorithm(algorithm string) string {
+	algorithm = hashAlgorithmToSpec(algorithm)
+	for _, recommAlgo := range RecommendedHashAlgorithms {
+		if algorithm == hashAlgorithmToSpec(recommAlgo) {
+			return recommAlgo
+		}
+	}
+	return algorithm
+}

modules/auth/password/hash/setting_test.go

@@ -0,0 +1,38 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package hash
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCheckSettingPasswordHashAlgorithm(t *testing.T) {
+	t.Run("pbkdf2 is pbkdf2_v2", func(t *testing.T) {
+		pbkdf2v2Config, pbkdf2v2Algo := SetDefaultPasswordHashAlgorithm("pbkdf2_v2")
+		pbkdf2Config, pbkdf2Algo := SetDefaultPasswordHashAlgorithm("pbkdf2")
+
+		assert.Equal(t, pbkdf2v2Config, pbkdf2Config)
+		assert.Equal(t, pbkdf2v2Algo.Specification, pbkdf2Algo.Specification)
+	})
+
+	for a, b := range aliasAlgorithmNames {
+		t.Run(a+"="+b, func(t *testing.T) {
+			aConfig, aAlgo := SetDefaultPasswordHashAlgorithm(a)
+			bConfig, bAlgo := SetDefaultPasswordHashAlgorithm(b)
+
+			assert.Equal(t, bConfig, aConfig)
+			assert.Equal(t, aAlgo.Specification, bAlgo.Specification)
+		})
+	}
+
+	t.Run("pbkdf2_v2 is the default when default password hash algorithm is empty", func(t *testing.T) {
+		emptyConfig, emptyAlgo := SetDefaultPasswordHashAlgorithm("")
+		pbkdf2v2Config, pbkdf2v2Algo := SetDefaultPasswordHashAlgorithm("pbkdf2_v2")
+
+		assert.Equal(t, pbkdf2v2Config, emptyConfig)
+		assert.Equal(t, pbkdf2v2Algo.Specification, emptyAlgo.Specification)
+	})
+}

modules/auth/password/password.go

@@ -0,0 +1,136 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package password
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"errors"
+	"html/template"
+	"math/big"
+	"strings"
+	"sync"
+
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/translation"
+)
+
+var (
+	ErrComplexity = errors.New("password not complex enough")
+	ErrMinLength  = errors.New("password not long enough")
+)
+
+// complexity contains information about a particular kind of password complexity
+type complexity struct {
+	ValidChars string
+	TrNameOne  string
+}
+
+var (
+	matchComplexityOnce sync.Once
+	validChars          string
+	requiredList        []complexity
+
+	charComplexities = map[string]complexity{
+		"lower": {
+			`abcdefghijklmnopqrstuvwxyz`,
+			"form.password_lowercase_one",
+		},
+		"upper": {
+			`ABCDEFGHIJKLMNOPQRSTUVWXYZ`,
+			"form.password_uppercase_one",
+		},
+		"digit": {
+			`0123456789`,
+			"form.password_digit_one",
+		},
+		"spec": {
+			` !"#$%&'()*+,-./:;<=>?@[\]^_{|}~` + "`",
+			"form.password_special_one",
+		},
+	}
+)
+
+// NewComplexity for preparation
+func NewComplexity() {
+	matchComplexityOnce.Do(func() {
+		setupComplexity(setting.PasswordComplexity)
+	})
+}
+
+func setupComplexity(values []string) {
+	if len(values) != 1 || values[0] != "off" {
+		for _, val := range values {
+			if complexity, ok := charComplexities[val]; ok {
+				validChars += complexity.ValidChars
+				requiredList = append(requiredList, complexity)
+			}
+		}
+		if len(requiredList) == 0 {
+			// No valid character classes found; use all classes as default
+			for _, complexity := range charComplexities {
+				validChars += complexity.ValidChars
+				requiredList = append(requiredList, complexity)
+			}
+		}
+	}
+	if validChars == "" {
+		// No complexities to check; provide a sensible default for password generation
+		validChars = charComplexities["lower"].ValidChars + charComplexities["upper"].ValidChars + charComplexities["digit"].ValidChars
+	}
+}
+
+// IsComplexEnough return True if password meets complexity settings
+func IsComplexEnough(pwd string) bool {
+	NewComplexity()
+	if len(validChars) > 0 {
+		for _, req := range requiredList {
+			if !strings.ContainsAny(req.ValidChars, pwd) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+// Generate a random password
+func Generate(n int) (string, error) {
+	NewComplexity()
+	buffer := make([]byte, n)
+	maxInt := big.NewInt(int64(len(validChars)))
+	for {
+		for j := range n {
+			rnd, err := rand.Int(rand.Reader, maxInt)
+			if err != nil {
+				return "", err
+			}
+			buffer[j] = validChars[rnd.Int64()]
+		}
+
+		if err := IsPwned(context.Background(), string(buffer)); err != nil {
+			if errors.Is(err, ErrIsPwned) {
+				continue
+			}
+			return "", err
+		}
+		if IsComplexEnough(string(buffer)) && string(buffer[0]) != " " && string(buffer[n-1]) != " " {
+			return string(buffer), nil
+		}
+	}
+}
+
+// BuildComplexityError builds the error message when password complexity checks fail
+func BuildComplexityError(locale translation.Locale) template.HTML {
+	var buffer bytes.Buffer
+	buffer.WriteString(locale.TrString("form.password_complexity"))
+	buffer.WriteString("<ul>")
+	for _, c := range requiredList {
+		buffer.WriteString("<li>")
+		buffer.WriteString(locale.TrString(c.TrNameOne))
+		buffer.WriteString("</li>")
+	}
+	buffer.WriteString("</ul>")
+	return template.HTML(buffer.String())
+}

modules/auth/password/password_test.go

@@ -0,0 +1,76 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package password
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestComplexity_IsComplexEnough(t *testing.T) {
+	matchComplexityOnce.Do(func() {})
+
+	testlist := []struct {
+		complexity  []string
+		truevalues  []string
+		falsevalues []string
+	}{
+		{[]string{"off"}, []string{"1", "-", "a", "A", "ñ", "日本語"}, []string{}},
+		{[]string{"lower"}, []string{"abc", "abc!"}, []string{"ABC", "123", "=!$", ""}},
+		{[]string{"upper"}, []string{"ABC"}, []string{"abc", "123", "=!$", "abc!", ""}},
+		{[]string{"digit"}, []string{"123"}, []string{"abc", "ABC", "=!$", "abc!", ""}},
+		{[]string{"spec"}, []string{"=!$", "abc!"}, []string{"abc", "ABC", "123", ""}},
+		{[]string{"off"}, []string{"abc", "ABC", "123", "=!$", "abc!", ""}, nil},
+		{[]string{"lower", "spec"}, []string{"abc!"}, []string{"abc", "ABC", "123", "=!$", "abcABC123", ""}},
+		{[]string{"lower", "upper", "digit"}, []string{"abcABC123"}, []string{"abc", "ABC", "123", "=!$", "abc!", ""}},
+		{[]string{""}, []string{"abC=1", "abc!9D"}, []string{"ABC", "123", "=!$", ""}},
+	}
+
+	for _, test := range testlist {
+		testComplextity(test.complexity)
+		for _, val := range test.truevalues {
+			assert.True(t, IsComplexEnough(val))
+		}
+		for _, val := range test.falsevalues {
+			assert.False(t, IsComplexEnough(val))
+		}
+	}
+
+	// Remove settings for other tests
+	testComplextity([]string{"off"})
+}
+
+func TestComplexity_Generate(t *testing.T) {
+	matchComplexityOnce.Do(func() {})
+
+	const maxCount = 50
+	const pwdLen = 50
+
+	test := func(t *testing.T, modes []string) {
+		testComplextity(modes)
+		for range maxCount {
+			pwd, err := Generate(pwdLen)
+			assert.NoError(t, err)
+			assert.Len(t, pwd, pwdLen)
+			assert.True(t, IsComplexEnough(pwd), "Failed complexities with modes %+v for generated: %s", modes, pwd)
+		}
+	}
+
+	test(t, []string{"lower"})
+	test(t, []string{"upper"})
+	test(t, []string{"lower", "upper", "spec"})
+	test(t, []string{"off"})
+	test(t, []string{""})
+
+	// Remove settings for other tests
+	testComplextity([]string{"off"})
+}
+
+func testComplextity(values []string) {
+	// Cleanup previous values
+	validChars = ""
+	requiredList = make([]complexity, 0, len(values))
+	setupComplexity(values)
+}

modules/auth/password/pwn.go

@@ -0,0 +1,52 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package password
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"code.gitea.io/gitea/modules/auth/password/pwn"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+var ErrIsPwned = errors.New("password has been pwned")
+
+type ErrIsPwnedRequest struct {
+	err error
+}
+
+func IsErrIsPwnedRequest(err error) bool {
+	_, ok := err.(ErrIsPwnedRequest)
+	return ok
+}
+
+func (err ErrIsPwnedRequest) Error() string {
+	return fmt.Sprintf("using Have-I-Been-Pwned service failed: %v", err.err)
+}
+
+func (err ErrIsPwnedRequest) Unwrap() error {
+	return err.err
+}
+
+// IsPwned checks whether a password has been pwned
+// If a password has not been pwned, no error is returned.
+func IsPwned(ctx context.Context, password string) error {
+	if !setting.PasswordCheckPwn {
+		return nil
+	}
+
+	client := pwn.New(pwn.WithContext(ctx))
+	count, err := client.CheckPassword(password, true)
+	if err != nil {
+		return ErrIsPwnedRequest{err}
+	}
+
+	if count > 0 {
+		return ErrIsPwned
+	}
+
+	return nil
+}

modules/auth/password/pwn/pwn.go

@@ -0,0 +1,118 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package pwn
+
+import (
+	"context"
+	"crypto/sha1"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/modules/setting"
+)
+
+const passwordURL = "https://api.pwnedpasswords.com/range/"
+
+// ErrEmptyPassword is an empty password error
+var ErrEmptyPassword = errors.New("password cannot be empty")
+
+// Client is a HaveIBeenPwned client
+type Client struct {
+	ctx  context.Context
+	http *http.Client
+}
+
+// New returns a new HaveIBeenPwned Client
+func New(options ...ClientOption) *Client {
+	client := &Client{
+		ctx:  context.Background(),
+		http: http.DefaultClient,
+	}
+
+	for _, opt := range options {
+		opt(client)
+	}
+
+	return client
+}
+
+// ClientOption is a way to modify a new Client
+type ClientOption func(*Client)
+
+// WithHTTP will set the http.Client of a Client
+func WithHTTP(httpClient *http.Client) func(pwnClient *Client) {
+	return func(pwnClient *Client) {
+		pwnClient.http = httpClient
+	}
+}
+
+// WithContext will set the context.Context of a Client
+func WithContext(ctx context.Context) func(pwnClient *Client) {
+	return func(pwnClient *Client) {
+		pwnClient.ctx = ctx
+	}
+}
+
+func newRequest(ctx context.Context, method, url string, body io.ReadCloser) (*http.Request, error) {
+	req, err := http.NewRequestWithContext(ctx, method, url, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("User-Agent", "Gitea "+setting.AppVer)
+	return req, nil
+}
+
+// CheckPassword returns the number of times a password has been compromised
+// Adding padding will make requests more secure, however is also slower
+// because artificial responses will be added to the response
+// For more information, see https://www.troyhunt.com/enhancing-pwned-passwords-privacy-with-padding/
+func (c *Client) CheckPassword(pw string, padding bool) (int, error) {
+	if pw == "" {
+		return -1, ErrEmptyPassword
+	}
+
+	sha := sha1.New()
+	sha.Write([]byte(pw))
+	enc := hex.EncodeToString(sha.Sum(nil))
+	prefix, suffix := enc[:5], enc[5:]
+
+	req, err := newRequest(c.ctx, http.MethodGet, fmt.Sprintf("%s%s", passwordURL, prefix), nil)
+	if err != nil {
+		return -1, nil
+	}
+	if padding {
+		req.Header.Add("Add-Padding", "true")
+	}
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return -1, err
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return -1, err
+	}
+	defer resp.Body.Close()
+
+	for pair := range strings.SplitSeq(string(body), "\n") {
+		parts := strings.Split(pair, ":")
+		if len(parts) != 2 {
+			continue
+		}
+		if strings.EqualFold(suffix, parts[0]) {
+			count, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64)
+			if err != nil {
+				return -1, err
+			}
+			return int(count), nil
+		}
+	}
+	return 0, nil
+}

modules/auth/password/pwn/pwn_test.go

@@ -0,0 +1,61 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package pwn
+
+import (
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type mockTransport struct{}
+
+func (mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if req.URL.Host != "api.pwnedpasswords.com" {
+		return nil, errors.New("unsupported host")
+	}
+	respMap := map[string]string{
+		"/range/5c1d8": "EAF2F254732680E8AC339B84F3266ECCBB5:1\r\nFC446EB88938834178CB9322C1EE273C2A7:2",
+		"/range/ba189": "FD4CB34F0378BCB15D23F6FFD28F0775C9E:3\r\nFDF342FCD8C3611DAE4D76E8A992A3E4169:4",
+		"/range/a1733": "C4CE0F1F0062B27B9E2F41AF0C08218017C:1\r\nFC446EB88938834178CB9322C1EE273C2A7:2\r\nFE81480327C992FE62065A827429DD1318B:0",
+		"/range/5617b": "FD4CB34F0378BCB15D23F6FFD28F0775C9E:3\r\nFDF342FCD8C3611DAE4D76E8A992A3E4169:4\r\nFE81480327C992FE62065A827429DD1318B:0",
+		"/range/79082": "FDF342FCD8C3611DAE4D76E8A992A3E4169:4\r\nFE81480327C992FE62065A827429DD1318B:0\r\nAFEF386F56EB0B4BE314E07696E5E6E6536:0",
+	}
+	if resp, ok := respMap[req.URL.Path]; ok {
+		return &http.Response{Request: req, Body: io.NopCloser(strings.NewReader(resp))}, nil
+	}
+	return nil, errors.New("unsupported path")
+}
+
+func TestPassword(t *testing.T) {
+	client := New(WithHTTP(&http.Client{Transport: mockTransport{}}))
+
+	count, err := client.CheckPassword("", false)
+	assert.ErrorIs(t, err, ErrEmptyPassword, "blank input should return ErrEmptyPassword")
+	assert.Equal(t, -1, count)
+
+	count, err = client.CheckPassword("pwned", false)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, count)
+
+	count, err = client.CheckPassword("notpwned", false)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, count)
+
+	count, err = client.CheckPassword("paddedpwned", true)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, count)
+
+	count, err = client.CheckPassword("paddednotpwned", true)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, count)
+
+	count, err = client.CheckPassword("paddednotpwnedzero", true)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, count)
+}

modules/auth/webauthn/webauthn.go

@@ -0,0 +1,80 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package webauthn
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/gob"
+
+	"code.gitea.io/gitea/models/auth"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+
+	"github.com/go-webauthn/webauthn/protocol"
+	"github.com/go-webauthn/webauthn/webauthn"
+)
+
+// WebAuthn represents the global WebAuthn instance
+var WebAuthn *webauthn.WebAuthn
+
+// Init initializes the WebAuthn instance from the config.
+func Init() {
+	gob.Register(&webauthn.SessionData{})
+
+	appURL, _ := protocol.FullyQualifiedOrigin(setting.AppURL)
+
+	WebAuthn = &webauthn.WebAuthn{
+		Config: &webauthn.Config{
+			RPDisplayName: setting.AppName,
+			RPID:          setting.Domain,
+			RPOrigins:     []string{appURL},
+			AuthenticatorSelection: protocol.AuthenticatorSelection{
+				UserVerification: protocol.VerificationDiscouraged,
+			},
+			AttestationPreference: protocol.PreferDirectAttestation,
+		},
+	}
+}
+
+// user represents an implementation of webauthn.User based on User model
+type user struct {
+	ctx  context.Context
+	User *user_model.User
+
+	defaultAuthFlags protocol.AuthenticatorFlags
+}
+
+var _ webauthn.User = (*user)(nil)
+
+func NewWebAuthnUser(ctx context.Context, u *user_model.User, defaultAuthFlags ...protocol.AuthenticatorFlags) webauthn.User {
+	return &user{ctx: ctx, User: u, defaultAuthFlags: util.OptionalArg(defaultAuthFlags)}
+}
+
+// WebAuthnID implements the webauthn.User interface
+func (u *user) WebAuthnID() []byte {
+	id := make([]byte, 8)
+	binary.PutVarint(id, u.User.ID)
+	return id
+}
+
+// WebAuthnName implements the webauthn.User interface
+func (u *user) WebAuthnName() string {
+	return util.IfZero(u.User.LoginName, u.User.Name)
+}
+
+// WebAuthnDisplayName implements the webauthn.User interface
+func (u *user) WebAuthnDisplayName() string {
+	return u.User.DisplayName()
+}
+
+// WebAuthnCredentials implements the webauthn.User interface
+func (u *user) WebAuthnCredentials() []webauthn.Credential {
+	dbCreds, err := auth.GetWebAuthnCredentialsByUID(u.ctx, u.User.ID)
+	if err != nil {
+		return nil
+	}
+	return dbCreds.ToCredentials(u.defaultAuthFlags)
+}

modules/auth/webauthn/webauthn_test.go

@@ -0,0 +1,25 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package webauthn
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInit(t *testing.T) {
+	setting.Domain = "domain"
+	setting.AppName = "AppName"
+	setting.AppURL = "https://domain/"
+	rpOrigin := []string{"https://domain"}
+
+	Init()
+
+	assert.Equal(t, setting.Domain, WebAuthn.Config.RPID)
+	assert.Equal(t, setting.AppName, WebAuthn.Config.RPDisplayName)
+	assert.Equal(t, rpOrigin, WebAuthn.Config.RPOrigins)
+}

modules/avatar/avatar.go

@@ -0,0 +1,139 @@
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package avatar
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"image"
+	"image/color"
+	"image/png"
+
+	_ "image/gif"  // for processing gif images
+	_ "image/jpeg" // for processing jpeg images
+
+	"code.gitea.io/gitea/modules/avatar/identicon"
+	"code.gitea.io/gitea/modules/setting"
+
+	"golang.org/x/image/draw"
+
+	_ "golang.org/x/image/webp" // for processing webp images
+)
+
+// DefaultAvatarSize is the target CSS pixel size for avatar generation. It is
+// multiplied by setting.Avatar.RenderedSizeFactor and the resulting size is the
+// usual size of avatar image saved on server, unless the original file is smaller
+// than the size after resizing.
+const DefaultAvatarSize = 256
+
+// RandomImageSize generates and returns a random avatar image unique to input data
+// in custom size (height and width).
+func RandomImageSize(size int, data []byte) (image.Image, error) {
+	// we use white as background, and use dark colors to draw blocks
+	imgMaker, err := identicon.New(size, color.White, identicon.DarkColors...)
+	if err != nil {
+		return nil, fmt.Errorf("identicon.New: %w", err)
+	}
+	return imgMaker.Make(data), nil
+}
+
+// RandomImage generates and returns a random avatar image unique to input data
+// in default size (height and width).
+func RandomImage(data []byte) (image.Image, error) {
+	return RandomImageSize(DefaultAvatarSize*setting.Avatar.RenderedSizeFactor, data)
+}
+
+// processAvatarImage process the avatar image data, crop and resize it if necessary.
+// the returned data could be the original image if no processing is needed.
+func processAvatarImage(data []byte, maxOriginSize int64) ([]byte, error) {
+	imgCfg, imgType, err := image.DecodeConfig(bytes.NewReader(data))
+	if err != nil {
+		return nil, fmt.Errorf("image.DecodeConfig: %w", err)
+	}
+
+	// for safety, only accept known types explicitly
+	if imgType != "png" && imgType != "jpeg" && imgType != "gif" && imgType != "webp" {
+		return nil, errors.New("unsupported avatar image type")
+	}
+
+	// do not process image which is too large, it would consume too much memory
+	if imgCfg.Width > setting.Avatar.MaxWidth {
+		return nil, fmt.Errorf("image width is too large: %d > %d", imgCfg.Width, setting.Avatar.MaxWidth)
+	}
+	if imgCfg.Height > setting.Avatar.MaxHeight {
+		return nil, fmt.Errorf("image height is too large: %d > %d", imgCfg.Height, setting.Avatar.MaxHeight)
+	}
+
+	// If the origin is small enough, just use it, then APNG could be supported,
+	// otherwise, if the image is processed later, APNG loses animation.
+	// And one more thing, webp is not fully supported, for animated webp, image.DecodeConfig works but Decode fails.
+	// So for animated webp, if the uploaded file is smaller than maxOriginSize, it will be used, if it's larger, there will be an error.
+	if len(data) < int(maxOriginSize) {
+		return data, nil
+	}
+
+	img, _, err := image.Decode(bytes.NewReader(data))
+	if err != nil {
+		return nil, fmt.Errorf("image.Decode: %w", err)
+	}
+
+	// try to crop and resize the origin image if necessary
+	img = cropSquare(img)
+
+	targetSize := DefaultAvatarSize * setting.Avatar.RenderedSizeFactor
+	img = scale(img, targetSize, targetSize, draw.BiLinear)
+
+	// try to encode the cropped/resized image to png
+	bs := bytes.Buffer{}
+	if err = png.Encode(&bs, img); err != nil {
+		return nil, err
+	}
+	resized := bs.Bytes()
+
+	// usually the png compression is not good enough, use the original image (no cropping/resizing) if the origin is smaller
+	if len(data) <= len(resized) {
+		return data, nil
+	}
+
+	return resized, nil
+}
+
+// ProcessAvatarImage process the avatar image data, crop and resize it if necessary.
+// the returned data could be the original image if no processing is needed.
+func ProcessAvatarImage(data []byte) ([]byte, error) {
+	return processAvatarImage(data, setting.Avatar.MaxOriginSize)
+}
+
+// scale resizes the image to width x height using the given scaler.
+func scale(src image.Image, width, height int, scale draw.Scaler) image.Image {
+	rect := image.Rect(0, 0, width, height)
+	dst := image.NewRGBA(rect)
+	scale.Scale(dst, rect, src, src.Bounds(), draw.Over, nil)
+	return dst
+}
+
+// cropSquare crops the largest square image from the center of the image.
+// If the image is already square, it is returned unchanged.
+func cropSquare(src image.Image) image.Image {
+	bounds := src.Bounds()
+	if bounds.Dx() == bounds.Dy() {
+		return src
+	}
+
+	var rect image.Rectangle
+	if bounds.Dx() > bounds.Dy() {
+		// width > height
+		size := bounds.Dy()
+		rect = image.Rect((bounds.Dx()-size)/2, 0, (bounds.Dx()+size)/2, size)
+	} else {
+		// width < height
+		size := bounds.Dx()
+		rect = image.Rect(0, (bounds.Dy()-size)/2, size, (bounds.Dy()+size)/2)
+	}
+
+	dst := image.NewRGBA(rect)
+	draw.Draw(dst, rect, src, rect.Min, draw.Src)
+	return dst
+}

modules/avatar/avatar_test.go

@@ -0,0 +1,136 @@
+// Copyright 2016 The Gogs Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package avatar
+
+import (
+	"bytes"
+	"image"
+	"image/png"
+	"os"
+	"testing"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_RandomImageSize(t *testing.T) {
+	_, err := RandomImageSize(0, []byte("gitea@local"))
+	assert.Error(t, err)
+
+	_, err = RandomImageSize(64, []byte("gitea@local"))
+	assert.NoError(t, err)
+}
+
+func Test_RandomImage(t *testing.T) {
+	_, err := RandomImage([]byte("gitea@local"))
+	assert.NoError(t, err)
+}
+
+func Test_ProcessAvatarPNG(t *testing.T) {
+	setting.Avatar.MaxWidth = 4096
+	setting.Avatar.MaxHeight = 4096
+
+	data, err := os.ReadFile("testdata/avatar.png")
+	assert.NoError(t, err)
+
+	_, err = processAvatarImage(data, 262144)
+	assert.NoError(t, err)
+}
+
+func Test_ProcessAvatarJPEG(t *testing.T) {
+	setting.Avatar.MaxWidth = 4096
+	setting.Avatar.MaxHeight = 4096
+
+	data, err := os.ReadFile("testdata/avatar.jpeg")
+	assert.NoError(t, err)
+
+	_, err = processAvatarImage(data, 262144)
+	assert.NoError(t, err)
+}
+
+func Test_ProcessAvatarInvalidData(t *testing.T) {
+	setting.Avatar.MaxWidth = 5
+	setting.Avatar.MaxHeight = 5
+
+	_, err := processAvatarImage([]byte{}, 12800)
+	assert.EqualError(t, err, "image.DecodeConfig: image: unknown format")
+}
+
+func Test_ProcessAvatarInvalidImageSize(t *testing.T) {
+	setting.Avatar.MaxWidth = 5
+	setting.Avatar.MaxHeight = 5
+
+	data, err := os.ReadFile("testdata/avatar.png")
+	assert.NoError(t, err)
+
+	_, err = processAvatarImage(data, 12800)
+	assert.EqualError(t, err, "image width is too large: 10 > 5")
+}
+
+func Test_ProcessAvatarImage(t *testing.T) {
+	setting.Avatar.MaxWidth = 4096
+	setting.Avatar.MaxHeight = 4096
+	scaledSize := DefaultAvatarSize * setting.Avatar.RenderedSizeFactor
+
+	newImgData := func(size int, optHeight ...int) []byte {
+		width := size
+		height := size
+		if len(optHeight) == 1 {
+			height = optHeight[0]
+		}
+		img := image.NewRGBA(image.Rect(0, 0, width, height))
+		bs := bytes.Buffer{}
+		err := png.Encode(&bs, img)
+		assert.NoError(t, err)
+		return bs.Bytes()
+	}
+
+	// if origin image canvas is too large, crop and resize it
+	origin := newImgData(500, 600)
+	result, err := processAvatarImage(origin, 0)
+	assert.NoError(t, err)
+	assert.NotEqual(t, origin, result)
+	decoded, err := png.Decode(bytes.NewReader(result))
+	assert.NoError(t, err)
+	assert.Equal(t, scaledSize, decoded.Bounds().Max.X)
+	assert.Equal(t, scaledSize, decoded.Bounds().Max.Y)
+
+	// if origin image is smaller than the default size, use the origin image
+	origin = newImgData(1)
+	result, err = processAvatarImage(origin, 0)
+	assert.NoError(t, err)
+	assert.Equal(t, origin, result)
+
+	// use the origin image if the origin is smaller
+	origin = newImgData(scaledSize + 100)
+	result, err = processAvatarImage(origin, 0)
+	assert.NoError(t, err)
+	assert.Less(t, len(result), len(origin))
+
+	// still use the origin image if the origin doesn't exceed the max-origin-size
+	origin = newImgData(scaledSize + 100)
+	result, err = processAvatarImage(origin, 262144)
+	assert.NoError(t, err)
+	assert.Equal(t, origin, result)
+
+	// allow to use known image format (eg: webp) if it is small enough
+	origin, err = os.ReadFile("testdata/animated.webp")
+	assert.NoError(t, err)
+	result, err = processAvatarImage(origin, 262144)
+	assert.NoError(t, err)
+	assert.Equal(t, origin, result)
+
+	// do not support unknown image formats, eg: SVG may contain embedded JS
+	origin = []byte("<svg></svg>")
+	_, err = processAvatarImage(origin, 262144)
+	assert.ErrorContains(t, err, "image: unknown format")
+
+	// make sure the canvas size limit works
+	setting.Avatar.MaxWidth = 5
+	setting.Avatar.MaxHeight = 5
+	origin = newImgData(10)
+	_, err = processAvatarImage(origin, 262144)
+	assert.ErrorContains(t, err, "image width is too large: 10 > 5")
+}

modules/avatar/hash.go

@@ -0,0 +1,28 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package avatar
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"strconv"
+)
+
+// HashAvatar will generate a unique string, which ensures that when there's a
+// different unique ID while the data is the same, it will generate a different
+// output. It will generate the output according to:
+// HEX(HASH(uniqueID || - || data))
+// The hash being used is SHA256.
+// The sole purpose of the unique ID is to generate a distinct hash Such that
+// two unique IDs with the same data will have a different hash output.
+// The "-" byte is important to ensure that data cannot be modified such that
+// the first byte is a number, which could lead to a "collision" with the hash
+// of another unique ID.
+func HashAvatar(uniqueID int64, data []byte) string {
+	h := sha256.New()
+	h.Write([]byte(strconv.FormatInt(uniqueID, 10)))
+	h.Write([]byte{'-'})
+	h.Write(data)
+	return hex.EncodeToString(h.Sum(nil))
+}

modules/avatar/hash_test.go

@@ -0,0 +1,26 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package avatar_test
+
+import (
+	"bytes"
+	"image"
+	"image/png"
+	"testing"
+
+	"code.gitea.io/gitea/modules/avatar"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_HashAvatar(t *testing.T) {
+	myImage := image.NewRGBA(image.Rect(0, 0, 32, 32))
+	var buff bytes.Buffer
+	png.Encode(&buff, myImage)
+
+	assert.Equal(t, "9ddb5bac41d57e72aa876321d0c09d71090c05f94bc625303801be2f3240d2cb", avatar.HashAvatar(1, buff.Bytes()))
+	assert.Equal(t, "9a5d44e5d637b9582a976676e8f3de1dccd877c2fe3e66ca3fab1629f2f47609", avatar.HashAvatar(8, buff.Bytes()))
+	assert.Equal(t, "ed7399158672088770de6f5211ce15528ebd675e92fc4fc060c025f4b2794ccb", avatar.HashAvatar(1024, buff.Bytes()))
+	assert.Equal(t, "161178642c7d59eb25a61dddced5e6b66eae1c70880d5f148b1b497b767e72d9", avatar.HashAvatar(1024, []byte{}))
+}

modules/avatar/identicon/block.go

@@ -0,0 +1,717 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+// Copied and modified from https://github.com/issue9/identicon/ (MIT License)
+
+package identicon
+
+import "image"
+
+var (
+	// the blocks can appear in center, these blocks can be more beautiful
+	centerBlocks = []blockFunc{b0, b1, b2, b3, b19, b26, b27}
+
+	// all blocks
+	blocks = []blockFunc{b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15, b16, b17, b18, b19, b20, b21, b22, b23, b24, b25, b26, b27}
+)
+
+type blockFunc func(img *image.Paletted, x, y, size, angle int)
+
+// draw a polygon by points, and the polygon is rotated by angle.
+func drawBlock(img *image.Paletted, x, y, size, angle int, points []int) {
+	if angle != 0 {
+		m := size / 2
+		rotate(points, m, m, angle)
+	}
+
+	for i := range size {
+		for j := range size {
+			if pointInPolygon(i, j, points) {
+				img.SetColorIndex(x+i, y+j, 1)
+			}
+		}
+	}
+}
+
+// blank
+//
+//	--------
+//	|      |
+//	|      |
+//	|      |
+//	--------
+func b0(img *image.Paletted, x, y, size, angle int) {}
+
+// full-filled
+//
+//	--------
+//	|######|
+//	|######|
+//	|######|
+//	--------
+func b1(img *image.Paletted, x, y, size, angle int) {
+	for i := x; i < x+size; i++ {
+		for j := y; j < y+size; j++ {
+			img.SetColorIndex(i, j, 1)
+		}
+	}
+}
+
+// a small block
+//
+//	----------
+//	|        |
+//	|  ####  |
+//	|  ####  |
+//	|        |
+//	----------
+func b2(img *image.Paletted, x, y, size, angle int) {
+	l := size / 4
+	x += l
+	y += l
+
+	for i := x; i < x+2*l; i++ {
+		for j := y; j < y+2*l; j++ {
+			img.SetColorIndex(i, j, 1)
+		}
+	}
+}
+
+// diamond
+//
+//	---------
+//	|   #   |
+//	|  ###  |
+//	| ##### |
+//	|#######|
+//	| ##### |
+//	|  ###  |
+//	|   #   |
+//	---------
+func b3(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	drawBlock(img, x, y, size, 0, []int{
+		m, 0,
+		size, m,
+		m, size,
+		0, m,
+		m, 0,
+	})
+}
+
+// b4
+//
+//	-------
+//	|#####|
+//	|#### |
+//	|###  |
+//	|##   |
+//	|#    |
+//	|------
+func b4(img *image.Paletted, x, y, size, angle int) {
+	drawBlock(img, x, y, size, angle, []int{
+		0, 0,
+		size, 0,
+		0, size,
+		0, 0,
+	})
+}
+
+// b5
+//
+//	---------
+//	|   #   |
+//	|  ###  |
+//	| ##### |
+//	|#######|
+func b5(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	drawBlock(img, x, y, size, angle, []int{
+		m, 0,
+		size, size,
+		0, size,
+		m, 0,
+	})
+}
+
+// b6
+//
+//	--------
+//	|###   |
+//	|###   |
+//	|###   |
+//	--------
+func b6(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	drawBlock(img, x, y, size, angle, []int{
+		0, 0,
+		m, 0,
+		m, size,
+		0, size,
+		0, 0,
+	})
+}
+
+// b7 italic cone
+//
+//	---------
+//	| #     |
+//	|  ##   |
+//	|  #####|
+//	|   ####|
+//	|--------
+func b7(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	drawBlock(img, x, y, size, angle, []int{
+		0, 0,
+		size, m,
+		size, size,
+		m, size,
+		0, 0,
+	})
+}
+
+// b8 three small triangles
+//
+//	-----------
+//	|    #    |
+//	|   ###   |
+//	|  #####  |
+//	|  #   #  |
+//	| ### ### |
+//	|#########|
+//	-----------
+func b8(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	mm := m / 2
+
+	// top
+	drawBlock(img, x, y, size, angle, []int{
+		m, 0,
+		3 * mm, m,
+		mm, m,
+		m, 0,
+	})
+
+	// bottom left
+	drawBlock(img, x, y, size, angle, []int{
+		mm, m,
+		m, size,
+		0, size,
+		mm, m,
+	})
+
+	// bottom right
+	drawBlock(img, x, y, size, angle, []int{
+		3 * mm, m,
+		size, size,
+		m, size,
+		3 * mm, m,
+	})
+}
+
+// b9 italic triangle
+//
+//	---------
+//	|#      |
+//	| ####  |
+//	|  #####|
+//	|  #### |
+//	|   #   |
+//	---------
+func b9(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	drawBlock(img, x, y, size, angle, []int{
+		0, 0,
+		size, m,
+		m, size,
+		0, 0,
+	})
+}
+
+// b10
+//
+//	----------
+//	|    ####|
+//	|    ### |
+//	|    ##  |
+//	|    #   |
+//	|####    |
+//	|###     |
+//	|##      |
+//	|#       |
+//	----------
+func b10(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	drawBlock(img, x, y, size, angle, []int{
+		m, 0,
+		size, 0,
+		m, m,
+		m, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		0, m,
+		m, m,
+		0, size,
+		0, m,
+	})
+}
+
+// b11
+//
+//	----------
+//	|####    |
+//	|####    |
+//	|####    |
+//	|        |
+//	|        |
+//	----------
+func b11(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	drawBlock(img, x, y, size, angle, []int{
+		0, 0,
+		m, 0,
+		m, m,
+		0, m,
+		0, 0,
+	})
+}
+
+// b12
+//
+//	-----------
+//	|         |
+//	|         |
+//	|#########|
+//	|  #####  |
+//	|    #    |
+//	-----------
+func b12(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	drawBlock(img, x, y, size, angle, []int{
+		0, m,
+		size, m,
+		m, size,
+		0, m,
+	})
+}
+
+// b13
+//
+//	-----------
+//	|         |
+//	|         |
+//	|    #    |
+//	|  #####  |
+//	|#########|
+//	-----------
+func b13(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	drawBlock(img, x, y, size, angle, []int{
+		m, m,
+		size, size,
+		0, size,
+		m, m,
+	})
+}
+
+// b14
+//
+//	---------
+//	|   #   |
+//	| ###   |
+//	|####   |
+//	|       |
+//	|       |
+//	---------
+func b14(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	drawBlock(img, x, y, size, angle, []int{
+		m, 0,
+		m, m,
+		0, m,
+		m, 0,
+	})
+}
+
+// b15
+//
+//	----------
+//	|#####   |
+//	|###     |
+//	|#       |
+//	|        |
+//	|        |
+//	----------
+func b15(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	drawBlock(img, x, y, size, angle, []int{
+		0, 0,
+		m, 0,
+		0, m,
+		0, 0,
+	})
+}
+
+// b16
+//
+//	---------
+//	|   #   |
+//	| ##### |
+//	|#######|
+//	|   #   |
+//	| ##### |
+//	|#######|
+//	---------
+func b16(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	drawBlock(img, x, y, size, angle, []int{
+		m, 0,
+		size, m,
+		0, m,
+		m, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		m, m,
+		size, size,
+		0, size,
+		m, m,
+	})
+}
+
+// b17
+//
+//	----------
+//	|#####   |
+//	|###     |
+//	|#       |
+//	|      ##|
+//	|      ##|
+//	----------
+func b17(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+
+	drawBlock(img, x, y, size, angle, []int{
+		0, 0,
+		m, 0,
+		0, m,
+		0, 0,
+	})
+
+	quarter := size / 4
+	drawBlock(img, x, y, size, angle, []int{
+		size - quarter, size - quarter,
+		size, size - quarter,
+		size, size,
+		size - quarter, size,
+		size - quarter, size - quarter,
+	})
+}
+
+// b18
+//
+//	----------
+//	|#####   |
+//	|####    |
+//	|###     |
+//	|##      |
+//	|#       |
+//	----------
+func b18(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+
+	drawBlock(img, x, y, size, angle, []int{
+		0, 0,
+		m, 0,
+		0, size,
+		0, 0,
+	})
+}
+
+// b19
+//
+//	----------
+//	|########|
+//	|###  ###|
+//	|#      #|
+//	|###  ###|
+//	|########|
+//	----------
+func b19(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+
+	drawBlock(img, x, y, size, angle, []int{
+		0, 0,
+		m, 0,
+		0, m,
+		0, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		m, 0,
+		size, 0,
+		size, m,
+		m, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		size, m,
+		size, size,
+		m, size,
+		size, m,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		0, m,
+		m, size,
+		0, size,
+		0, m,
+	})
+}
+
+// b20
+//
+//	----------
+//	|  ##     |
+//	|###      |
+//	|##       |
+//	|##       |
+//	|#        |
+//	----------
+func b20(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	q := size / 4
+
+	drawBlock(img, x, y, size, angle, []int{
+		q, 0,
+		0, size,
+		0, m,
+		q, 0,
+	})
+}
+
+// b21
+//
+//	----------
+//	| ####   |
+//	|## #####|
+//	|##    ##|
+//	|##      |
+//	|#       |
+//	----------
+func b21(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	q := size / 4
+
+	drawBlock(img, x, y, size, angle, []int{
+		q, 0,
+		0, size,
+		0, m,
+		q, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		q, 0,
+		size, q,
+		size, m,
+		q, 0,
+	})
+}
+
+// b22
+//
+//	----------
+//	| ####   |
+//	|##  ### |
+//	|##    ##|
+//	|##    ##|
+//	|#      #|
+//	----------
+func b22(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	q := size / 4
+
+	drawBlock(img, x, y, size, angle, []int{
+		q, 0,
+		0, size,
+		0, m,
+		q, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		q, 0,
+		size, q,
+		size, size,
+		q, 0,
+	})
+}
+
+// b23
+//
+//	----------
+//	| #######|
+//	|###    #|
+//	|##      |
+//	|##      |
+//	|#       |
+//	----------
+func b23(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	q := size / 4
+
+	drawBlock(img, x, y, size, angle, []int{
+		q, 0,
+		0, size,
+		0, m,
+		q, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		q, 0,
+		size, 0,
+		size, q,
+		q, 0,
+	})
+}
+
+// b24
+//
+//	----------
+//	| ##  ###|
+//	|###  ###|
+//	|##  ##  |
+//	|##  ##  |
+//	|#   #   |
+//	----------
+func b24(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	q := size / 4
+
+	drawBlock(img, x, y, size, angle, []int{
+		q, 0,
+		0, size,
+		0, m,
+		q, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		m, 0,
+		size, 0,
+		m, size,
+		m, 0,
+	})
+}
+
+// b25
+//
+//	----------
+//	|#      #|
+//	|##   ###|
+//	|##  ##  |
+//	|######  |
+//	|####    |
+//	----------
+func b25(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	q := size / 4
+
+	drawBlock(img, x, y, size, angle, []int{
+		0, 0,
+		0, size,
+		q, size,
+		0, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		0, m,
+		size, 0,
+		q, size,
+		0, m,
+	})
+}
+
+// b26
+//
+//	----------
+//	|#      #|
+//	|###  ###|
+//	|  ####  |
+//	|###  ###|
+//	|#      #|
+//	----------
+func b26(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	q := size / 4
+
+	drawBlock(img, x, y, size, angle, []int{
+		0, 0,
+		m, q,
+		q, m,
+		0, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		size, 0,
+		m + q, m,
+		m, q,
+		size, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		size, size,
+		m, m + q,
+		q + m, m,
+		size, size,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		0, size,
+		q, m,
+		m, q + m,
+		0, size,
+	})
+}
+
+// b27
+//
+//	----------
+//	|########|
+//	|##   ###|
+//	|#      #|
+//	|###   ##|
+//	|########|
+//	----------
+func b27(img *image.Paletted, x, y, size, angle int) {
+	m := size / 2
+	q := size / 4
+
+	drawBlock(img, x, y, size, angle, []int{
+		0, 0,
+		size, 0,
+		0, q,
+		0, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		q + m, 0,
+		size, 0,
+		size, size,
+		q + m, 0,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		size, q + m,
+		size, size,
+		0, size,
+		size, q + m,
+	})
+
+	drawBlock(img, x, y, size, angle, []int{
+		0, size,
+		0, 0,
+		q, size,
+		0, size,
+	})
+}

modules/avatar/identicon/colors.go

@@ -0,0 +1,134 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package identicon
+
+import "image/color"
+
+// DarkColors are dark colors for avatar blocks, they come from image/color/palette.WebSafe, and light colors (0xff) are removed
+var DarkColors = []color.Color{
+	color.RGBA{0x00, 0x00, 0x33, 0xff},
+	color.RGBA{0x00, 0x00, 0x66, 0xff},
+	color.RGBA{0x00, 0x00, 0x99, 0xff},
+	color.RGBA{0x00, 0x00, 0xcc, 0xff},
+	color.RGBA{0x00, 0x33, 0x00, 0xff},
+	color.RGBA{0x00, 0x33, 0x33, 0xff},
+	color.RGBA{0x00, 0x33, 0x66, 0xff},
+	color.RGBA{0x00, 0x33, 0x99, 0xff},
+	color.RGBA{0x00, 0x33, 0xcc, 0xff},
+	color.RGBA{0x00, 0x66, 0x00, 0xff},
+	color.RGBA{0x00, 0x66, 0x33, 0xff},
+	color.RGBA{0x00, 0x66, 0x66, 0xff},
+	color.RGBA{0x00, 0x66, 0x99, 0xff},
+	color.RGBA{0x00, 0x66, 0xcc, 0xff},
+	color.RGBA{0x00, 0x99, 0x00, 0xff},
+	color.RGBA{0x00, 0x99, 0x33, 0xff},
+	color.RGBA{0x00, 0x99, 0x66, 0xff},
+	color.RGBA{0x00, 0x99, 0x99, 0xff},
+	color.RGBA{0x00, 0x99, 0xcc, 0xff},
+	color.RGBA{0x00, 0xcc, 0x00, 0xff},
+	color.RGBA{0x00, 0xcc, 0x33, 0xff},
+	color.RGBA{0x00, 0xcc, 0x66, 0xff},
+	color.RGBA{0x00, 0xcc, 0x99, 0xff},
+	color.RGBA{0x00, 0xcc, 0xcc, 0xff},
+	color.RGBA{0x33, 0x00, 0x00, 0xff},
+	color.RGBA{0x33, 0x00, 0x33, 0xff},
+	color.RGBA{0x33, 0x00, 0x66, 0xff},
+	color.RGBA{0x33, 0x00, 0x99, 0xff},
+	color.RGBA{0x33, 0x00, 0xcc, 0xff},
+	color.RGBA{0x33, 0x33, 0x00, 0xff},
+	color.RGBA{0x33, 0x33, 0x33, 0xff},
+	color.RGBA{0x33, 0x33, 0x66, 0xff},
+	color.RGBA{0x33, 0x33, 0x99, 0xff},
+	color.RGBA{0x33, 0x33, 0xcc, 0xff},
+	color.RGBA{0x33, 0x66, 0x00, 0xff},
+	color.RGBA{0x33, 0x66, 0x33, 0xff},
+	color.RGBA{0x33, 0x66, 0x66, 0xff},
+	color.RGBA{0x33, 0x66, 0x99, 0xff},
+	color.RGBA{0x33, 0x66, 0xcc, 0xff},
+	color.RGBA{0x33, 0x99, 0x00, 0xff},
+	color.RGBA{0x33, 0x99, 0x33, 0xff},
+	color.RGBA{0x33, 0x99, 0x66, 0xff},
+	color.RGBA{0x33, 0x99, 0x99, 0xff},
+	color.RGBA{0x33, 0x99, 0xcc, 0xff},
+	color.RGBA{0x33, 0xcc, 0x00, 0xff},
+	color.RGBA{0x33, 0xcc, 0x33, 0xff},
+	color.RGBA{0x33, 0xcc, 0x66, 0xff},
+	color.RGBA{0x33, 0xcc, 0x99, 0xff},
+	color.RGBA{0x33, 0xcc, 0xcc, 0xff},
+	color.RGBA{0x66, 0x00, 0x00, 0xff},
+	color.RGBA{0x66, 0x00, 0x33, 0xff},
+	color.RGBA{0x66, 0x00, 0x66, 0xff},
+	color.RGBA{0x66, 0x00, 0x99, 0xff},
+	color.RGBA{0x66, 0x00, 0xcc, 0xff},
+	color.RGBA{0x66, 0x33, 0x00, 0xff},
+	color.RGBA{0x66, 0x33, 0x33, 0xff},
+	color.RGBA{0x66, 0x33, 0x66, 0xff},
+	color.RGBA{0x66, 0x33, 0x99, 0xff},
+	color.RGBA{0x66, 0x33, 0xcc, 0xff},
+	color.RGBA{0x66, 0x66, 0x00, 0xff},
+	color.RGBA{0x66, 0x66, 0x33, 0xff},
+	color.RGBA{0x66, 0x66, 0x66, 0xff},
+	color.RGBA{0x66, 0x66, 0x99, 0xff},
+	color.RGBA{0x66, 0x66, 0xcc, 0xff},
+	color.RGBA{0x66, 0x99, 0x00, 0xff},
+	color.RGBA{0x66, 0x99, 0x33, 0xff},
+	color.RGBA{0x66, 0x99, 0x66, 0xff},
+	color.RGBA{0x66, 0x99, 0x99, 0xff},
+	color.RGBA{0x66, 0x99, 0xcc, 0xff},
+	color.RGBA{0x66, 0xcc, 0x00, 0xff},
+	color.RGBA{0x66, 0xcc, 0x33, 0xff},
+	color.RGBA{0x66, 0xcc, 0x66, 0xff},
+	color.RGBA{0x66, 0xcc, 0x99, 0xff},
+	color.RGBA{0x66, 0xcc, 0xcc, 0xff},
+	color.RGBA{0x99, 0x00, 0x00, 0xff},
+	color.RGBA{0x99, 0x00, 0x33, 0xff},
+	color.RGBA{0x99, 0x00, 0x66, 0xff},
+	color.RGBA{0x99, 0x00, 0x99, 0xff},
+	color.RGBA{0x99, 0x00, 0xcc, 0xff},
+	color.RGBA{0x99, 0x33, 0x00, 0xff},
+	color.RGBA{0x99, 0x33, 0x33, 0xff},
+	color.RGBA{0x99, 0x33, 0x66, 0xff},
+	color.RGBA{0x99, 0x33, 0x99, 0xff},
+	color.RGBA{0x99, 0x33, 0xcc, 0xff},
+	color.RGBA{0x99, 0x66, 0x00, 0xff},
+	color.RGBA{0x99, 0x66, 0x33, 0xff},
+	color.RGBA{0x99, 0x66, 0x66, 0xff},
+	color.RGBA{0x99, 0x66, 0x99, 0xff},
+	color.RGBA{0x99, 0x66, 0xcc, 0xff},
+	color.RGBA{0x99, 0x99, 0x00, 0xff},
+	color.RGBA{0x99, 0x99, 0x33, 0xff},
+	color.RGBA{0x99, 0x99, 0x66, 0xff},
+	color.RGBA{0x99, 0x99, 0x99, 0xff},
+	color.RGBA{0x99, 0x99, 0xcc, 0xff},
+	color.RGBA{0x99, 0xcc, 0x00, 0xff},
+	color.RGBA{0x99, 0xcc, 0x33, 0xff},
+	color.RGBA{0x99, 0xcc, 0x66, 0xff},
+	color.RGBA{0x99, 0xcc, 0x99, 0xff},
+	color.RGBA{0x99, 0xcc, 0xcc, 0xff},
+	color.RGBA{0xcc, 0x00, 0x00, 0xff},
+	color.RGBA{0xcc, 0x00, 0x33, 0xff},
+	color.RGBA{0xcc, 0x00, 0x66, 0xff},
+	color.RGBA{0xcc, 0x00, 0x99, 0xff},
+	color.RGBA{0xcc, 0x00, 0xcc, 0xff},
+	color.RGBA{0xcc, 0x33, 0x00, 0xff},
+	color.RGBA{0xcc, 0x33, 0x33, 0xff},
+	color.RGBA{0xcc, 0x33, 0x66, 0xff},
+	color.RGBA{0xcc, 0x33, 0x99, 0xff},
+	color.RGBA{0xcc, 0x33, 0xcc, 0xff},
+	color.RGBA{0xcc, 0x66, 0x00, 0xff},
+	color.RGBA{0xcc, 0x66, 0x33, 0xff},
+	color.RGBA{0xcc, 0x66, 0x66, 0xff},
+	color.RGBA{0xcc, 0x66, 0x99, 0xff},
+	color.RGBA{0xcc, 0x66, 0xcc, 0xff},
+	color.RGBA{0xcc, 0x99, 0x00, 0xff},
+	color.RGBA{0xcc, 0x99, 0x33, 0xff},
+	color.RGBA{0xcc, 0x99, 0x66, 0xff},
+	color.RGBA{0xcc, 0x99, 0x99, 0xff},
+	color.RGBA{0xcc, 0x99, 0xcc, 0xff},
+	color.RGBA{0xcc, 0xcc, 0x00, 0xff},
+	color.RGBA{0xcc, 0xcc, 0x33, 0xff},
+	color.RGBA{0xcc, 0xcc, 0x66, 0xff},
+	color.RGBA{0xcc, 0xcc, 0x99, 0xff},
+	color.RGBA{0xcc, 0xcc, 0xcc, 0xff},
+}

modules/avatar/identicon/identicon.go

@@ -0,0 +1,141 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+// Copied and modified from https://github.com/issue9/identicon/ (MIT License)
+// Generate pseudo-random avatars by IP, E-mail, etc.
+
+package identicon
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"image"
+	"image/color"
+)
+
+const minImageSize = 16
+
+// Identicon is used to generate pseudo-random avatars
+type Identicon struct {
+	foreColors []color.Color
+	backColor  color.Color
+	size       int
+	rect       image.Rectangle
+}
+
+// New returns an Identicon struct with the correct settings
+// size image size
+// back background color
+// fore all possible foreground colors. only one foreground color will be picked randomly for one image
+func New(size int, back color.Color, fore ...color.Color) (*Identicon, error) {
+	if len(fore) == 0 {
+		return nil, errors.New("foreground is not set")
+	}
+
+	if size < minImageSize {
+		return nil, fmt.Errorf("size %d is smaller than min size %d", size, minImageSize)
+	}
+
+	return &Identicon{
+		foreColors: fore,
+		backColor:  back,
+		size:       size,
+		rect:       image.Rect(0, 0, size, size),
+	}, nil
+}
+
+// Make generates an avatar by data
+func (i *Identicon) Make(data []byte) image.Image {
+	h := sha256.New()
+	h.Write(data)
+	sum := h.Sum(nil)
+
+	b1 := int(sum[0]+sum[1]+sum[2]) % len(blocks)
+	b2 := int(sum[3]+sum[4]+sum[5]) % len(blocks)
+	c := int(sum[6]+sum[7]+sum[8]) % len(centerBlocks)
+	b1Angle := int(sum[9]+sum[10]) % 4
+	b2Angle := int(sum[11]+sum[12]) % 4
+	foreColor := int(sum[11]+sum[12]+sum[15]) % len(i.foreColors)
+
+	return i.render(c, b1, b2, b1Angle, b2Angle, foreColor)
+}
+
+func (i *Identicon) render(c, b1, b2, b1Angle, b2Angle, foreColor int) image.Image {
+	p := image.NewPaletted(i.rect, []color.Color{i.backColor, i.foreColors[foreColor]})
+	drawBlocks(p, i.size, centerBlocks[c], blocks[b1], blocks[b2], b1Angle, b2Angle)
+	return p
+}
+
+/*
+# Algorithm
+
+Origin: An image is split into 9 areas
+
+```
+  -------------
+  | 1 | 2 | 3 |
+  -------------
+  | 4 | 5 | 6 |
+  -------------
+  | 7 | 8 | 9 |
+  -------------
+```
+
+Area 1/3/9/7 use a 90-degree rotating pattern.
+Area 1/3/9/7 use another 90-degree rotating pattern.
+Area 5 uses a random pattern.
+
+The Patched Fix: make the image left-right mirrored to get rid of something like "swastika"
+*/
+
+// draw blocks to the paletted
+// c: the block drawer for the center block
+// b1,b2: the block drawers for other blocks (around the center block)
+// b1Angle,b2Angle: the angle for the rotation of b1/b2
+func drawBlocks(p *image.Paletted, size int, c, b1, b2 blockFunc, b1Angle, b2Angle int) {
+	nextAngle := func(a int) int {
+		return (a + 1) % 4
+	}
+
+	padding := (size % 3) / 2 // in cased the size can not be aligned by 3 blocks.
+
+	blockSize := size / 3
+	twoBlockSize := 2 * blockSize
+
+	// center
+	c(p, blockSize+padding, blockSize+padding, blockSize, 0)
+
+	// left top (1)
+	b1(p, 0+padding, 0+padding, blockSize, b1Angle)
+	// center top (2)
+	b2(p, blockSize+padding, 0+padding, blockSize, b2Angle)
+
+	b1Angle = nextAngle(b1Angle)
+	b2Angle = nextAngle(b2Angle)
+	// right top (3)
+	// b1(p, twoBlockSize+padding, 0+padding, blockSize, b1Angle)
+	// right middle (6)
+	// b2(p, twoBlockSize+padding, blockSize+padding, blockSize, b2Angle)
+
+	b1Angle = nextAngle(b1Angle)
+	b2Angle = nextAngle(b2Angle)
+	// right bottom (9)
+	// b1(p, twoBlockSize+padding, twoBlockSize+padding, blockSize, b1Angle)
+	// center bottom (8)
+	b2(p, blockSize+padding, twoBlockSize+padding, blockSize, b2Angle)
+
+	b1Angle = nextAngle(b1Angle)
+	b2Angle = nextAngle(b2Angle)
+	// lef bottom (7)
+	b1(p, 0+padding, twoBlockSize+padding, blockSize, b1Angle)
+	// left middle (4)
+	b2(p, 0+padding, blockSize+padding, blockSize, b2Angle)
+
+	// then we make it left-right mirror, so we didn't draw 3/6/9 before
+	for x := 0; x < size/2; x++ {
+		for y := range size {
+			p.SetColorIndex(size-x, y, p.ColorIndexAt(x, y))
+		}
+	}
+}

modules/avatar/identicon/identicon_test.go

@@ -0,0 +1,40 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build test_avatar_identicon
+
+package identicon
+
+import (
+	"image/color"
+	"image/png"
+	"os"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGenerate(t *testing.T) {
+	dir, _ := os.Getwd()
+	dir = dir + "/testdata"
+	if st, err := os.Stat(dir); err != nil || !st.IsDir() {
+		t.Errorf("can not save generated images to %s", dir)
+	}
+
+	backColor := color.White
+	imgMaker, err := New(64, backColor, DarkColors...)
+	assert.NoError(t, err)
+	for i := 0; i < 100; i++ {
+		s := strconv.Itoa(i)
+		img := imgMaker.Make([]byte(s))
+
+		f, err := os.Create(dir + "/" + s + ".png")
+		if !assert.NoError(t, err) {
+			continue
+		}
+		defer f.Close()
+		err = png.Encode(f, img)
+		assert.NoError(t, err)
+	}
+}

modules/avatar/identicon/polygon.go

@@ -0,0 +1,68 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+// Copied and modified from https://github.com/issue9/identicon/ (MIT License)
+
+package identicon
+
+var (
+	// cos(0),cos(90),cos(180),cos(270)
+	cos = []int{1, 0, -1, 0}
+
+	// sin(0),sin(90),sin(180),sin(270)
+	sin = []int{0, 1, 0, -1}
+)
+
+// rotate the points by center point (x,y)
+// angle: [0,1,2,3] means [0，90，180，270] degree
+func rotate(points []int, x, y, angle int) {
+	// the angle is only used internally, and it has been guaranteed to be 0/1/2/3, so we do not check it again
+	for i := 0; i < len(points); i += 2 {
+		px, py := points[i]-x, points[i+1]-y
+		points[i] = px*cos[angle] - py*sin[angle] + x
+		points[i+1] = px*sin[angle] + py*cos[angle] + y
+	}
+}
+
+// check whether the point is inside the polygon (defined by the points)
+// the first and the last point must be the same
+func pointInPolygon(x, y int, polygonPoints []int) bool {
+	if len(polygonPoints) < 8 { // a valid polygon must have more than 2 points
+		return false
+	}
+
+	// reference: nonzero winding rule, https://en.wikipedia.org/wiki/Nonzero-rule
+	// split the plane into two by the check point horizontally:
+	//   y>0，includes (x>0 && y==0)
+	//   y<0，includes (x<0 && y==0)
+	//
+	// then scan every point in the polygon.
+	//
+	// if current point and previous point are in different planes (eg: curY>0 && prevY<0),
+	// check the clock-direction from previous point to current point (use check point as origin).
+	// if the direction is clockwise, then r++, otherwise then r--
+	// finally, if 2==abs(r), then the check point is inside the polygon
+
+	r := 0
+	prevX, prevY := polygonPoints[0], polygonPoints[1]
+	prev := (prevY > y) || ((prevX > x) && (prevY == y))
+	for i := 2; i < len(polygonPoints); i += 2 {
+		currX, currY := polygonPoints[i], polygonPoints[i+1]
+		curr := (currY > y) || ((currX > x) && (currY == y))
+
+		if curr == prev {
+			prevX, prevY = currX, currY
+			continue
+		}
+
+		if mul := (prevX-x)*(currY-y) - (currX-x)*(prevY-y); mul >= 0 {
+			r++
+		} else { // mul < 0
+			r--
+		}
+		prevX, prevY = currX, currY
+		prev = curr
+	}
+
+	return r == 2 || r == -2
+}

modules/badge/badge.go

@@ -0,0 +1,129 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package badge
+
+import (
+	"strings"
+	"sync"
+	"unicode"
+
+	actions_model "code.gitea.io/gitea/models/actions"
+)
+
+// The Badge layout: |offset|label|message|
+// We use 10x scale to calculate more precisely
+// Then scale down to normal size in tmpl file
+
+type Text struct {
+	text  string
+	width int
+	x     int
+}
+
+func (t Text) Text() string {
+	return t.text
+}
+
+func (t Text) Width() int {
+	return t.width
+}
+
+func (t Text) X() int {
+	return t.x
+}
+
+func (t Text) TextLength() int {
+	return int(float64(t.width-defaultOffset) * 10)
+}
+
+type Badge struct {
+	IDPrefix   string
+	FontFamily string
+	Color      string
+	FontSize   int
+	Label      Text
+	Message    Text
+}
+
+func (b Badge) Width() int {
+	return b.Label.width + b.Message.width
+}
+
+// Style follows https://shields.io/badges
+const (
+	StyleFlat       = "flat"
+	StyleFlatSquare = "flat-square"
+)
+
+const (
+	defaultOffset     = 10
+	defaultFontSize   = 11
+	DefaultColor      = "#9f9f9f" // Grey
+	DefaultFontFamily = "DejaVu Sans,Verdana,Geneva,sans-serif"
+	DefaultStyle      = StyleFlat
+)
+
+var GlobalVars = sync.OnceValue(func() (ret struct {
+	StatusColorMap       map[actions_model.Status]string
+	DejaVuGlyphWidthData map[rune]uint8
+	AllStyles            []string
+},
+) {
+	ret.StatusColorMap = map[actions_model.Status]string{
+		actions_model.StatusSuccess:   "#4c1",    // Green
+		actions_model.StatusSkipped:   "#dfb317", // Yellow
+		actions_model.StatusUnknown:   "#97ca00", // Light Green
+		actions_model.StatusFailure:   "#e05d44", // Red
+		actions_model.StatusCancelled: "#fe7d37", // Orange
+		actions_model.StatusWaiting:   "#dfb317", // Yellow
+		actions_model.StatusRunning:   "#dfb317", // Yellow
+		actions_model.StatusBlocked:   "#dfb317", // Yellow
+	}
+	ret.DejaVuGlyphWidthData = dejaVuGlyphWidthDataFunc()
+	ret.AllStyles = []string{StyleFlat, StyleFlatSquare}
+	return ret
+})
+
+// GenerateBadge generates badge with given template
+func GenerateBadge(label, message, color string) Badge {
+	lw := calculateTextWidth(label) + defaultOffset
+	mw := calculateTextWidth(message) + defaultOffset
+
+	lx := lw * 5
+	mx := lw*10 + mw*5 - 10
+	return Badge{
+		FontFamily: DefaultFontFamily,
+		Label: Text{
+			text:  label,
+			width: lw,
+			x:     lx,
+		},
+		Message: Text{
+			text:  message,
+			width: mw,
+			x:     mx,
+		},
+		FontSize: defaultFontSize * 10,
+		Color:    color,
+	}
+}
+
+func calculateTextWidth(text string) int {
+	width := 0
+	widthData := GlobalVars().DejaVuGlyphWidthData
+	for _, char := range strings.TrimSpace(text) {
+		charWidth, ok := widthData[char]
+		if !ok {
+			// use the width of 'm' in case of missing glyph width data for a printable character
+			if unicode.IsPrint(char) {
+				charWidth = widthData['m']
+			} else {
+				charWidth = 0
+			}
+		}
+		width += int(charWidth)
+	}
+
+	return width
+}

modules/badge/badge_glyph_width.go

@@ -0,0 +1,206 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package badge
+
+// DejaVuGlyphWidthData is generated by `sfnt.Face.GlyphAdvance(nil, <rune>, 11, font.HintingNone)` with DejaVu Sans
+// v2.37 (https://github.com/dejavu-fonts/dejavu-fonts/releases/download/version_2_37/dejavu-sans-ttf-2.37.zip).
+//
+// Fonts defined in "DefaultFontFamily" all have similar widths (including "DejaVu Sans"),
+// and these widths are fixed and don't seem to change.
+//
+// A devtest page "/devtest/badge-actions-svg" could be used to check the rendered images.
+
+func dejaVuGlyphWidthDataFunc() map[rune]uint8 {
+	return map[rune]uint8{
+		32:  3,
+		33:  4,
+		34:  5,
+		35:  9,
+		36:  7,
+		37:  10,
+		38:  9,
+		39:  3,
+		40:  4,
+		41:  4,
+		42:  6,
+		43:  9,
+		44:  3,
+		45:  4,
+		46:  3,
+		47:  4,
+		48:  7,
+		49:  7,
+		50:  7,
+		51:  7,
+		52:  7,
+		53:  7,
+		54:  7,
+		55:  7,
+		56:  7,
+		57:  7,
+		58:  4,
+		59:  4,
+		60:  9,
+		61:  9,
+		62:  9,
+		63:  6,
+		64:  11,
+		65:  8,
+		66:  8,
+		67:  8,
+		68:  8,
+		69:  7,
+		70:  6,
+		71:  9,
+		72:  8,
+		73:  3,
+		74:  3,
+		75:  7,
+		76:  6,
+		77:  9,
+		78:  8,
+		79:  9,
+		80:  7,
+		81:  9,
+		82:  8,
+		83:  7,
+		84:  7,
+		85:  8,
+		86:  8,
+		87:  11,
+		88:  8,
+		89:  7,
+		90:  8,
+		91:  4,
+		92:  4,
+		93:  4,
+		94:  9,
+		95:  6,
+		96:  6,
+		97:  7,
+		98:  7,
+		99:  6,
+		100: 7,
+		101: 7,
+		102: 4,
+		103: 7,
+		104: 7,
+		105: 3,
+		106: 3,
+		107: 6,
+		108: 3,
+		109: 11,
+		110: 7,
+		111: 7,
+		112: 7,
+		113: 7,
+		114: 5,
+		115: 6,
+		116: 4,
+		117: 7,
+		118: 7,
+		119: 9,
+		120: 7,
+		121: 7,
+		122: 6,
+		123: 7,
+		124: 4,
+		125: 7,
+		126: 9,
+		161: 4,
+		162: 7,
+		163: 7,
+		164: 7,
+		165: 7,
+		166: 4,
+		167: 6,
+		168: 6,
+		169: 11,
+		170: 5,
+		171: 7,
+		172: 9,
+		174: 11,
+		175: 6,
+		176: 6,
+		177: 9,
+		178: 4,
+		179: 4,
+		180: 6,
+		181: 7,
+		182: 7,
+		183: 3,
+		184: 6,
+		185: 4,
+		186: 5,
+		187: 7,
+		188: 11,
+		189: 11,
+		190: 11,
+		191: 6,
+		192: 8,
+		193: 8,
+		194: 8,
+		195: 8,
+		196: 8,
+		197: 8,
+		198: 11,
+		199: 8,
+		200: 7,
+		201: 7,
+		202: 7,
+		203: 7,
+		204: 3,
+		205: 3,
+		206: 3,
+		207: 3,
+		208: 9,
+		209: 8,
+		210: 9,
+		211: 9,
+		212: 9,
+		213: 9,
+		214: 9,
+		215: 9,
+		216: 9,
+		217: 8,
+		218: 8,
+		219: 8,
+		220: 8,
+		221: 7,
+		222: 7,
+		223: 7,
+		224: 7,
+		225: 7,
+		226: 7,
+		227: 7,
+		228: 7,
+		229: 7,
+		230: 11,
+		231: 6,
+		232: 7,
+		233: 7,
+		234: 7,
+		235: 7,
+		236: 3,
+		237: 3,
+		238: 3,
+		239: 3,
+		240: 7,
+		241: 7,
+		242: 7,
+		243: 7,
+		244: 7,
+		245: 7,
+		246: 7,
+		247: 9,
+		248: 7,
+		249: 7,
+		250: 7,
+		251: 7,
+		252: 7,
+		253: 7,
+		254: 7,
+		255: 7,
+	}
+}

modules/base/natural_sort.go

@@ -0,0 +1,70 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package base
+
+import (
+	"unicode/utf8"
+
+	"golang.org/x/text/collate"
+	"golang.org/x/text/language"
+)
+
+func naturalSortGetRune(str string, pos int) (r rune, size int, has bool) {
+	if pos >= len(str) {
+		return 0, 0, false
+	}
+	r, size = utf8.DecodeRuneInString(str[pos:])
+	if r == utf8.RuneError {
+		r, size = rune(str[pos]), 1 // if invalid input, treat it as a single byte ascii
+	}
+	return r, size, true
+}
+
+func naturalSortAdvance(str string, pos int) (end int, isNumber bool) {
+	end = pos
+	for {
+		r, size, has := naturalSortGetRune(str, end)
+		if !has {
+			break
+		}
+		isCurRuneNum := '0' <= r && r <= '9'
+		if end == pos {
+			isNumber = isCurRuneNum
+			end += size
+		} else if isCurRuneNum == isNumber {
+			end += size
+		} else {
+			break
+		}
+	}
+	return end, isNumber
+}
+
+// NaturalSortLess compares two strings so that they could be sorted in natural order
+func NaturalSortLess(s1, s2 string) bool {
+	// There is a bug in Golang's collate package: https://github.com/golang/go/issues/67997
+	// text/collate: CompareString(collate.Numeric) returns wrong result for "0.0" vs "1.0" #67997
+	// So we need to handle the number parts by ourselves
+	c := collate.New(language.English, collate.Numeric)
+	pos1, pos2 := 0, 0
+	for pos1 < len(s1) && pos2 < len(s2) {
+		end1, isNum1 := naturalSortAdvance(s1, pos1)
+		end2, isNum2 := naturalSortAdvance(s2, pos2)
+		part1, part2 := s1[pos1:end1], s2[pos2:end2]
+		if isNum1 && isNum2 {
+			if part1 != part2 {
+				if len(part1) != len(part2) {
+					return len(part1) < len(part2)
+				}
+				return part1 < part2
+			}
+		} else {
+			if cmp := c.CompareString(part1, part2); cmp != 0 {
+				return cmp < 0
+			}
+		}
+		pos1, pos2 = end1, end2
+	}
+	return len(s1) < len(s2)
+}

modules/base/natural_sort_test.go

@@ -0,0 +1,45 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package base
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNaturalSortLess(t *testing.T) {
+	testLess := func(s1, s2 string) {
+		assert.True(t, NaturalSortLess(s1, s2), "s1<s2 should be true: s1=%q, s2=%q", s1, s2)
+		assert.False(t, NaturalSortLess(s2, s1), "s2<s1 should be false: s1=%q, s2=%q", s1, s2)
+	}
+	testEqual := func(s1, s2 string) {
+		assert.False(t, NaturalSortLess(s1, s2), "s1<s2 should be false: s1=%q, s2=%q", s1, s2)
+		assert.False(t, NaturalSortLess(s2, s1), "s2<s1 should be false: s1=%q, s2=%q", s1, s2)
+	}
+
+	testEqual("", "")
+	testLess("", "a")
+	testLess("", "1")
+
+	testLess("v1.2", "v1.2.0")
+	testLess("v1.2.0", "v1.10.0")
+	testLess("v1.20.0", "v1.29.0")
+	testEqual("v1.20.0", "v1.20.0")
+
+	testLess("a", "A")
+	testLess("a", "B")
+	testLess("A", "b")
+	testLess("A", "ab")
+
+	testLess("abc", "bcd")
+	testLess("a-1-a", "a-1-b")
+	testLess("2", "12")
+
+	testLess("cafe", "café")
+	testLess("café", "caff")
+
+	testLess("A-2", "A-11")
+	testLess("0.txt", "1.txt")
+}

modules/base/tool.go

@@ -0,0 +1,124 @@
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package base
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"strconv"
+	"time"
+
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+
+	"github.com/dustin/go-humanize"
+)
+
+// EncodeSha256 string to sha256 hex value.
+func EncodeSha256(str string) string {
+	h := sha256.New()
+	_, _ = h.Write([]byte(str))
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+// ShortSha is basically just truncating.
+// It is DEPRECATED and will be removed in the future.
+func ShortSha(sha1 string) string {
+	return util.TruncateRunes(sha1, 10)
+}
+
+// VerifyTimeLimitCode verify time limit code
+func VerifyTimeLimitCode(now time.Time, data string, minutes int, code string) bool {
+	if len(code) <= 18 {
+		return false
+	}
+
+	startTimeStr := code[:12]
+	aliveTimeStr := code[12:18]
+	aliveTime, _ := strconv.Atoi(aliveTimeStr) // no need to check err, if anything wrong, the following code check will fail soon
+
+	// check code
+	retCode := CreateTimeLimitCode(data, aliveTime, startTimeStr, nil)
+	if subtle.ConstantTimeCompare([]byte(retCode), []byte(code)) != 1 {
+		return false
+	}
+
+	// check time is expired or not: startTime <= now && now < startTime + minutes
+	startTime, _ := time.ParseInLocation("200601021504", startTimeStr, time.Local)
+	return (startTime.Before(now) || startTime.Equal(now)) && now.Before(startTime.Add(time.Minute*time.Duration(minutes)))
+}
+
+// TimeLimitCodeLength default value for time limit code
+const TimeLimitCodeLength = 12 + 6 + 40
+
+// CreateTimeLimitCode create a time-limited code.
+// Format: 12 length date time string + 6 minutes string (not used) + 40 hash string, some other code depends on this fixed length
+// If h is nil, then use the default hmac hash.
+func CreateTimeLimitCode[T time.Time | string](data string, minutes int, startTimeGeneric T, h hash.Hash) string {
+	const format = "200601021504"
+
+	var start time.Time
+	var startTimeAny any = startTimeGeneric
+	if t, ok := startTimeAny.(time.Time); ok {
+		start = t
+	} else {
+		var err error
+		start, err = time.ParseInLocation(format, startTimeAny.(string), time.Local)
+		if err != nil {
+			return "" // return an invalid code because the "parse" failed
+		}
+	}
+	startStr := start.Format(format)
+	end := start.Add(time.Minute * time.Duration(minutes))
+
+	if h == nil {
+		h = hmac.New(sha1.New, setting.GetGeneralTokenSigningSecret())
+	}
+	_, _ = fmt.Fprintf(h, "%s%s%s%s%d", data, hex.EncodeToString(setting.GetGeneralTokenSigningSecret()), startStr, end.Format(format), minutes)
+	encoded := hex.EncodeToString(h.Sum(nil))
+
+	code := fmt.Sprintf("%s%06d%s", startStr, minutes, encoded)
+	if len(code) != TimeLimitCodeLength {
+		panic("there is a hard requirement for the length of time-limited code") // it shouldn't happen
+	}
+	return code
+}
+
+// FileSize calculates the file size and generate user-friendly string.
+func FileSize(s int64) string {
+	return humanize.IBytes(uint64(s))
+}
+
+// StringsToInt64s converts a slice of string to a slice of int64.
+func StringsToInt64s(strs []string) ([]int64, error) {
+	if strs == nil {
+		return nil, nil
+	}
+	ints := make([]int64, 0, len(strs))
+	for _, s := range strs {
+		if s == "" {
+			continue
+		}
+		n, err := strconv.ParseInt(s, 10, 64)
+		if err != nil {
+			return nil, err
+		}
+		ints = append(ints, n)
+	}
+	return ints, nil
+}
+
+// Int64sToStrings converts a slice of int64 to a slice of string.
+func Int64sToStrings(ints []int64) []string {
+	strs := make([]string, len(ints))
+	for i := range ints {
+		strs[i] = strconv.FormatInt(ints[i], 10)
+	}
+	return strs
+}

modules/base/tool_test.go

@@ -0,0 +1,117 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package base
+
+import (
+	"crypto/sha1"
+	"fmt"
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/test"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEncodeSha256(t *testing.T) {
+	assert.Equal(t,
+		"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
+		EncodeSha256("foobar"),
+	)
+}
+
+func TestShortSha(t *testing.T) {
+	assert.Equal(t, "veryverylo", ShortSha("veryverylong"))
+}
+
+func TestVerifyTimeLimitCode(t *testing.T) {
+	defer test.MockVariableValue(&setting.InstallLock, true)()
+	initGeneralSecret := func(secret string) {
+		setting.InstallLock = true
+		setting.CfgProvider, _ = setting.NewConfigProviderFromData(fmt.Sprintf(`
+[oauth2]
+JWT_SECRET = %s
+`, secret))
+		setting.LoadCommonSettings()
+	}
+
+	initGeneralSecret("KZb_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko")
+	now := time.Now()
+
+	t.Run("TestGenericParameter", func(t *testing.T) {
+		time2000 := time.Date(2000, 1, 2, 3, 4, 5, 0, time.Local)
+		assert.Equal(t, "2000010203040000026fa5221b2731b7cf80b1b506f5e39e38c115fee5", CreateTimeLimitCode("test-sha1", 2, time2000, sha1.New()))
+		assert.Equal(t, "2000010203040000026fa5221b2731b7cf80b1b506f5e39e38c115fee5", CreateTimeLimitCode("test-sha1", 2, "200001020304", sha1.New()))
+		assert.Equal(t, "2000010203040000024842227a2f87041ff82025199c0187410a9297bf", CreateTimeLimitCode("test-hmac", 2, time2000, nil))
+		assert.Equal(t, "2000010203040000024842227a2f87041ff82025199c0187410a9297bf", CreateTimeLimitCode("test-hmac", 2, "200001020304", nil))
+	})
+
+	t.Run("TestInvalidCode", func(t *testing.T) {
+		assert.False(t, VerifyTimeLimitCode(now, "data", 2, ""))
+		assert.False(t, VerifyTimeLimitCode(now, "data", 2, "invalid code"))
+	})
+
+	t.Run("TestCreateAndVerify", func(t *testing.T) {
+		code := CreateTimeLimitCode("data", 2, now, nil)
+		assert.False(t, VerifyTimeLimitCode(now.Add(-time.Minute), "data", 2, code)) // not started yet
+		assert.True(t, VerifyTimeLimitCode(now, "data", 2, code))
+		assert.True(t, VerifyTimeLimitCode(now.Add(time.Minute), "data", 2, code))
+		assert.False(t, VerifyTimeLimitCode(now.Add(time.Minute), "DATA", 2, code))   // invalid data
+		assert.False(t, VerifyTimeLimitCode(now.Add(2*time.Minute), "data", 2, code)) // expired
+	})
+
+	t.Run("TestDifferentSecret", func(t *testing.T) {
+		// use another secret to ensure the code is invalid for different secret
+		verifyDataCode := func(c string) bool {
+			return VerifyTimeLimitCode(now, "data", 2, c)
+		}
+		code := CreateTimeLimitCode("data", 2, now, nil)
+		assert.True(t, verifyDataCode(code))
+		initGeneralSecret("000_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko")
+		assert.False(t, verifyDataCode(code))
+	})
+}
+
+func TestFileSize(t *testing.T) {
+	var size int64 = 512
+	assert.Equal(t, "512 B", FileSize(size))
+	size *= 1024
+	assert.Equal(t, "512 KiB", FileSize(size))
+	size *= 1024
+	assert.Equal(t, "512 MiB", FileSize(size))
+	size *= 1024
+	assert.Equal(t, "512 GiB", FileSize(size))
+	size *= 1024
+	assert.Equal(t, "512 TiB", FileSize(size))
+	size *= 1024
+	assert.Equal(t, "512 PiB", FileSize(size))
+	size *= 4
+	assert.Equal(t, "2.0 EiB", FileSize(size))
+}
+
+func TestStringsToInt64s(t *testing.T) {
+	testSuccess := func(input []string, expected []int64) {
+		result, err := StringsToInt64s(input)
+		assert.NoError(t, err)
+		assert.Equal(t, expected, result)
+	}
+	testSuccess(nil, nil)
+	testSuccess([]string{}, []int64{})
+	testSuccess([]string{""}, []int64{})
+	testSuccess([]string{"-1234"}, []int64{-1234})
+	testSuccess([]string{"1", "4", "16", "64", "256"}, []int64{1, 4, 16, 64, 256})
+
+	ints, err := StringsToInt64s([]string{"-1", "a"})
+	assert.Empty(t, ints)
+	assert.Error(t, err)
+}
+
+func TestInt64sToStrings(t *testing.T) {
+	assert.Equal(t, []string{}, Int64sToStrings([]int64{}))
+	assert.Equal(t,
+		[]string{"1", "4", "16", "64", "256"},
+		Int64sToStrings([]int64{1, 4, 16, 64, 256}),
+	)
+}

modules/cache/cache.go

@@ -0,0 +1,119 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cache
+
+import (
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strconv"
+	"time"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	_ "gitea.com/go-chi/cache/memcache" //nolint:depguard // memcache plugin for cache, it is required for config "ADAPTER=memcache"
+)
+
+var defaultCache StringCache
+
+// Init start cache service
+func Init() error {
+	if defaultCache == nil {
+		c, err := NewStringCache(setting.CacheService.Cache)
+		if err != nil {
+			return err
+		}
+		for range 10 {
+			if err = c.Ping(); err == nil {
+				break
+			}
+			time.Sleep(time.Second)
+		}
+		if err != nil {
+			return err
+		}
+		defaultCache = c
+	}
+	return nil
+}
+
+const (
+	testCacheKey = "DefaultCache.TestKey"
+	// SlowCacheThreshold marks cache tests as slow
+	// set to 30ms per discussion: https://github.com/go-gitea/gitea/issues/33190
+	// TODO: Replace with metrics histogram
+	SlowCacheThreshold = 30 * time.Millisecond
+)
+
+// Test performs delete, put and get operations on a predefined key
+// returns
+func Test() (time.Duration, error) {
+	if defaultCache == nil {
+		return 0, errors.New("default cache not initialized")
+	}
+
+	testData := hex.EncodeToString(make([]byte, 500))
+
+	start := time.Now()
+
+	if err := defaultCache.Delete(testCacheKey); err != nil {
+		return 0, fmt.Errorf("expect cache to delete data based on key if exist but got: %w", err)
+	}
+	if err := defaultCache.Put(testCacheKey, testData, 10); err != nil {
+		return 0, fmt.Errorf("expect cache to store data but got: %w", err)
+	}
+	testVal, hit := defaultCache.Get(testCacheKey)
+	if !hit {
+		return 0, errors.New("expect cache hit but got none")
+	}
+	if testVal != testData {
+		return 0, errors.New("expect cache to return same value as stored but got other")
+	}
+
+	return time.Since(start), nil
+}
+
+// GetCache returns the currently configured cache
+func GetCache() StringCache {
+	return defaultCache
+}
+
+// GetString returns the key value from cache with callback when no key exists in cache
+func GetString(key string, getFunc func() (string, error)) (string, error) {
+	if defaultCache == nil || setting.CacheService.TTL == 0 {
+		return getFunc()
+	}
+	cached, exist := defaultCache.Get(key)
+	if !exist {
+		value, err := getFunc()
+		if err != nil {
+			return value, err
+		}
+		return value, defaultCache.Put(key, value, setting.CacheService.TTLSeconds())
+	}
+	return cached, nil
+}
+
+// GetInt64 returns key value from cache with callback when no key exists in cache
+func GetInt64(key string, getFunc func() (int64, error)) (int64, error) {
+	s, err := GetString(key, func() (string, error) {
+		v, err := getFunc()
+		return strconv.FormatInt(v, 10), err
+	})
+	if err != nil {
+		return 0, err
+	}
+	if s == "" {
+		return 0, nil
+	}
+	return strconv.ParseInt(s, 10, 64)
+}
+
+// Remove key from cache
+func Remove(key string) {
+	if defaultCache == nil {
+		return
+	}
+	_ = defaultCache.Delete(key)
+}

modules/cache/cache_redis.go

@@ -0,0 +1,162 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cache
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/nosql"
+
+	"gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here
+	"github.com/redis/go-redis/v9"
+)
+
+// RedisCacher represents a redis cache adapter implementation.
+type RedisCacher struct {
+	c          redis.UniversalClient
+	prefix     string
+	hsetName   string
+	occupyMode bool
+}
+
+// toStr convert string/int/int64 interface to string. it's only used by the RedisCacher.Put internally
+func toStr(v any) string {
+	if v == nil {
+		return ""
+	}
+	switch v := v.(type) {
+	case string:
+		return v
+	case []byte:
+		return string(v)
+	case int:
+		return strconv.FormatInt(int64(v), 10)
+	case int64:
+		return strconv.FormatInt(v, 10)
+	default:
+		return fmt.Sprint(v) // as what the old com.ToStr does in most cases
+	}
+}
+
+// Put puts value (string type) into cache with key and expire time.
+// If expired is 0, it lives forever.
+func (c *RedisCacher) Put(key string, val any, expire int64) error {
+	// this function is not well-designed, it only puts string values into cache
+	key = c.prefix + key
+	if expire == 0 {
+		if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), 0).Err(); err != nil {
+			return err
+		}
+	} else {
+		dur := time.Duration(expire) * time.Second
+		if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), dur).Err(); err != nil {
+			return err
+		}
+	}
+
+	if c.occupyMode {
+		return nil
+	}
+	return c.c.HSet(graceful.GetManager().HammerContext(), c.hsetName, key, "0").Err()
+}
+
+// Get gets cached value by given key.
+func (c *RedisCacher) Get(key string) any {
+	val, err := c.c.Get(graceful.GetManager().HammerContext(), c.prefix+key).Result()
+	if err != nil {
+		return nil
+	}
+	return val
+}
+
+// Delete deletes cached value by given key.
+func (c *RedisCacher) Delete(key string) error {
+	key = c.prefix + key
+	if err := c.c.Del(graceful.GetManager().HammerContext(), key).Err(); err != nil {
+		return err
+	}
+
+	if c.occupyMode {
+		return nil
+	}
+	return c.c.HDel(graceful.GetManager().HammerContext(), c.hsetName, key).Err()
+}
+
+// Incr increases cached int-type value by given key as a counter.
+func (c *RedisCacher) Incr(key string) error {
+	if !c.IsExist(key) {
+		return fmt.Errorf("key '%s' not exist", key)
+	}
+	return c.c.Incr(graceful.GetManager().HammerContext(), c.prefix+key).Err()
+}
+
+// Decr decreases cached int-type value by given key as a counter.
+func (c *RedisCacher) Decr(key string) error {
+	if !c.IsExist(key) {
+		return fmt.Errorf("key '%s' not exist", key)
+	}
+	return c.c.Decr(graceful.GetManager().HammerContext(), c.prefix+key).Err()
+}
+
+// IsExist returns true if cached value exists.
+func (c *RedisCacher) IsExist(key string) bool {
+	if c.c.Exists(graceful.GetManager().HammerContext(), c.prefix+key).Val() == 1 {
+		return true
+	}
+
+	if !c.occupyMode {
+		c.c.HDel(graceful.GetManager().HammerContext(), c.hsetName, c.prefix+key)
+	}
+	return false
+}
+
+// Flush deletes all cached data.
+func (c *RedisCacher) Flush() error {
+	if c.occupyMode {
+		return c.c.FlushDB(graceful.GetManager().HammerContext()).Err()
+	}
+
+	keys, err := c.c.HKeys(graceful.GetManager().HammerContext(), c.hsetName).Result()
+	if err != nil {
+		return err
+	}
+	if err = c.c.Del(graceful.GetManager().HammerContext(), keys...).Err(); err != nil {
+		return err
+	}
+	return c.c.Del(graceful.GetManager().HammerContext(), c.hsetName).Err()
+}
+
+// StartAndGC starts GC routine based on config string settings.
+// AdapterConfig: network=tcp,addr=:6379,password=macaron,db=0,pool_size=100,idle_timeout=180,hset_name=MacaronCache,prefix=cache:
+func (c *RedisCacher) StartAndGC(opts cache.Options) error {
+	c.hsetName = "MacaronCache"
+	c.occupyMode = opts.OccupyMode
+
+	uri := nosql.ToRedisURI(opts.AdapterConfig)
+
+	c.c = nosql.GetManager().GetRedisClient(uri.String())
+
+	for k, v := range uri.Query() {
+		switch k {
+		case "hset_name":
+			c.hsetName = v[0]
+		case "prefix":
+			c.prefix = v[0]
+		}
+	}
+
+	return c.c.Ping(graceful.GetManager().HammerContext()).Err()
+}
+
+// Ping tests if the cache is alive.
+func (c *RedisCacher) Ping() error {
+	return c.c.Ping(graceful.GetManager().HammerContext()).Err()
+}
+
+func init() {
+	cache.Register("redis", &RedisCacher{})
+}

modules/cache/cache_test.go

@@ -0,0 +1,126 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cache
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func createTestCache() {
+	defaultCache, _ = NewStringCache(setting.Cache{
+		Adapter: "memory",
+		TTL:     time.Minute,
+	})
+	setting.CacheService.TTL = 24 * time.Hour
+}
+
+func TestNewContext(t *testing.T) {
+	assert.NoError(t, Init())
+
+	setting.CacheService.Cache = setting.Cache{Adapter: "redis", Conn: "some random string"}
+	con, err := NewStringCache(setting.Cache{
+		Adapter:  "rand",
+		Conn:     "false conf",
+		Interval: 100,
+	})
+	assert.Error(t, err)
+	assert.Nil(t, con)
+}
+
+func TestTest(t *testing.T) {
+	defaultCache = nil
+	_, err := Test()
+	assert.Error(t, err)
+
+	createTestCache()
+	elapsed, err := Test()
+	assert.NoError(t, err)
+	// mem cache should take from 300ns up to 1ms on modern hardware ...
+	assert.Positive(t, elapsed)
+	assert.Less(t, elapsed, SlowCacheThreshold)
+}
+
+func TestGetCache(t *testing.T) {
+	createTestCache()
+
+	assert.NotNil(t, GetCache())
+}
+
+func TestGetString(t *testing.T) {
+	createTestCache()
+
+	data, err := GetString("key", func() (string, error) {
+		return "", errors.New("some error")
+	})
+	assert.Error(t, err)
+	assert.Empty(t, data)
+
+	data, err = GetString("key", func() (string, error) {
+		return "", nil
+	})
+	assert.NoError(t, err)
+	assert.Empty(t, data)
+
+	data, err = GetString("key", func() (string, error) {
+		return "some data", nil
+	})
+	assert.NoError(t, err)
+	assert.Empty(t, data)
+	Remove("key")
+
+	data, err = GetString("key", func() (string, error) {
+		return "some data", nil
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, "some data", data)
+
+	data, err = GetString("key", func() (string, error) {
+		return "", errors.New("some error")
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, "some data", data)
+	Remove("key")
+}
+
+func TestGetInt64(t *testing.T) {
+	createTestCache()
+
+	data, err := GetInt64("key", func() (int64, error) {
+		return 0, errors.New("some error")
+	})
+	assert.Error(t, err)
+	assert.EqualValues(t, 0, data)
+
+	data, err = GetInt64("key", func() (int64, error) {
+		return 0, nil
+	})
+	assert.NoError(t, err)
+	assert.EqualValues(t, 0, data)
+
+	data, err = GetInt64("key", func() (int64, error) {
+		return 100, nil
+	})
+	assert.NoError(t, err)
+	assert.EqualValues(t, 0, data)
+	Remove("key")
+
+	data, err = GetInt64("key", func() (int64, error) {
+		return 100, nil
+	})
+	assert.NoError(t, err)
+	assert.EqualValues(t, 100, data)
+
+	data, err = GetInt64("key", func() (int64, error) {
+		return 0, errors.New("some error")
+	})
+	assert.NoError(t, err)
+	assert.EqualValues(t, 100, data)
+	Remove("key")
+}

modules/cache/cache_twoqueue.go

@@ -0,0 +1,208 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cache
+
+import (
+	"strconv"
+	"sync"
+	"time"
+
+	"code.gitea.io/gitea/modules/json"
+
+	mc "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here
+	lru "github.com/hashicorp/golang-lru/v2"
+)
+
+// TwoQueueCache represents a LRU 2Q cache adapter implementation
+type TwoQueueCache struct {
+	lock     sync.Mutex
+	cache    *lru.TwoQueueCache[string, any]
+	interval int
+}
+
+// TwoQueueCacheConfig describes the configuration for TwoQueueCache
+type TwoQueueCacheConfig struct {
+	Size        int     `ini:"SIZE" json:"size"`
+	RecentRatio float64 `ini:"RECENT_RATIO" json:"recent_ratio"`
+	GhostRatio  float64 `ini:"GHOST_RATIO" json:"ghost_ratio"`
+}
+
+// MemoryItem represents a memory cache item.
+type MemoryItem struct {
+	Val     any
+	Created int64
+	Timeout int64
+}
+
+func (item *MemoryItem) hasExpired() bool {
+	return item.Timeout > 0 &&
+		(time.Now().Unix()-item.Created) >= item.Timeout
+}
+
+var _ mc.Cache = &TwoQueueCache{}
+
+// Put puts value into cache with key and expire time.
+func (c *TwoQueueCache) Put(key string, val any, timeout int64) error {
+	item := &MemoryItem{
+		Val:     val,
+		Created: time.Now().Unix(),
+		Timeout: timeout,
+	}
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.cache.Add(key, item)
+	return nil
+}
+
+// Get gets cached value by given key.
+func (c *TwoQueueCache) Get(key string) any {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	cached, ok := c.cache.Get(key)
+	if !ok {
+		return nil
+	}
+	item, ok := cached.(*MemoryItem)
+
+	if !ok || item.hasExpired() {
+		c.cache.Remove(key)
+		return nil
+	}
+
+	return item.Val
+}
+
+// Delete deletes cached value by given key.
+func (c *TwoQueueCache) Delete(key string) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.cache.Remove(key)
+	return nil
+}
+
+// Incr increases cached int-type value by given key as a counter.
+func (c *TwoQueueCache) Incr(key string) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	cached, ok := c.cache.Get(key)
+	if !ok {
+		return nil
+	}
+	item, ok := cached.(*MemoryItem)
+
+	if !ok || item.hasExpired() {
+		c.cache.Remove(key)
+		return nil
+	}
+
+	var err error
+	item.Val, err = mc.Incr(item.Val)
+	return err
+}
+
+// Decr decreases cached int-type value by given key as a counter.
+func (c *TwoQueueCache) Decr(key string) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	cached, ok := c.cache.Get(key)
+	if !ok {
+		return nil
+	}
+	item, ok := cached.(*MemoryItem)
+
+	if !ok || item.hasExpired() {
+		c.cache.Remove(key)
+		return nil
+	}
+
+	var err error
+	item.Val, err = mc.Decr(item.Val)
+	return err
+}
+
+// IsExist returns true if cached value exists.
+func (c *TwoQueueCache) IsExist(key string) bool {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	cached, ok := c.cache.Peek(key)
+	if !ok {
+		return false
+	}
+	item, ok := cached.(*MemoryItem)
+	if !ok || item.hasExpired() {
+		c.cache.Remove(key)
+		return false
+	}
+
+	return true
+}
+
+// Flush deletes all cached data.
+func (c *TwoQueueCache) Flush() error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.cache.Purge()
+	return nil
+}
+
+func (c *TwoQueueCache) checkAndInvalidate(key string) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	cached, ok := c.cache.Peek(key)
+	if !ok {
+		return
+	}
+	item, ok := cached.(*MemoryItem)
+	if !ok || item.hasExpired() {
+		c.cache.Remove(key)
+	}
+}
+
+func (c *TwoQueueCache) startGC() {
+	if c.interval < 0 {
+		return
+	}
+	for _, key := range c.cache.Keys() {
+		c.checkAndInvalidate(key)
+	}
+	time.AfterFunc(time.Duration(c.interval)*time.Second, c.startGC)
+}
+
+// StartAndGC starts GC routine based on config string settings.
+func (c *TwoQueueCache) StartAndGC(opts mc.Options) error {
+	var err error
+	size := 50000
+	if opts.AdapterConfig != "" {
+		size, err = strconv.Atoi(opts.AdapterConfig)
+	}
+	if err != nil {
+		if !json.Valid([]byte(opts.AdapterConfig)) {
+			return err
+		}
+
+		cfg := &TwoQueueCacheConfig{
+			Size:        50000,
+			RecentRatio: lru.Default2QRecentRatio,
+			GhostRatio:  lru.Default2QGhostEntries,
+		}
+		_ = json.Unmarshal([]byte(opts.AdapterConfig), cfg)
+		c.cache, err = lru.New2QParams[string, any](cfg.Size, cfg.RecentRatio, cfg.GhostRatio)
+	} else {
+		c.cache, err = lru.New2Q[string, any](size)
+	}
+	c.interval = opts.Interval
+	if c.interval > 0 {
+		go c.startGC()
+	}
+	return err
+}
+
+// Ping tests if the cache is alive.
+func (c *TwoQueueCache) Ping() error {
+	return mc.GenericPing(c)
+}
+
+func init() {
+	mc.Register("twoqueue", &TwoQueueCache{})
+}

modules/cache/context.go

@@ -0,0 +1,43 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cache
+
+import (
+	"context"
+	"time"
+)
+
+type cacheContextKeyType struct{}
+
+var cacheContextKey = cacheContextKeyType{}
+
+// contextCacheLifetime is the max lifetime of context cache.
+// Since context cache is used to cache data in a request level context, 5 minutes is enough.
+// If a context cache is used more than 5 minutes, it's probably abused.
+const contextCacheLifetime = 5 * time.Minute
+
+func WithCacheContext(ctx context.Context) context.Context {
+	if c := GetContextCache(ctx); c != nil {
+		return ctx
+	}
+	return context.WithValue(ctx, cacheContextKey, NewEphemeralCache(contextCacheLifetime))
+}
+
+func GetContextCache(ctx context.Context) *EphemeralCache {
+	c, _ := ctx.Value(cacheContextKey).(*EphemeralCache)
+	return c
+}
+
+// GetWithContextCache returns the cache value of the given key in the given context.
+// FIXME: in some cases, the "context cache" should not be used, because it has uncontrollable behaviors
+// For example, these calls:
+// * GetWithContextCache(TargetID) -> OtherCodeCreateModel(TargetID) -> GetWithContextCache(TargetID)
+// Will cause the second call is not able to get the correct created target.
+// UNLESS it is certain that the target won't be changed during the request, DO NOT use it.
+func GetWithContextCache[T, K any](ctx context.Context, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) {
+	if c := GetContextCache(ctx); c != nil {
+		return GetWithEphemeralCache(ctx, c, groupKey, targetKey, f)
+	}
+	return f(ctx, targetKey)
+}

modules/cache/context_test.go

@@ -0,0 +1,50 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cache
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/modules/test"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWithCacheContext(t *testing.T) {
+	ctx := WithCacheContext(t.Context())
+	c := GetContextCache(ctx)
+	v, _ := c.Get("empty_field", "my_config1")
+	assert.Nil(t, v)
+
+	const field = "system_setting"
+	v, _ = c.Get(field, "my_config1")
+	assert.Nil(t, v)
+	c.Put(field, "my_config1", 1)
+	v, _ = c.Get(field, "my_config1")
+	assert.NotNil(t, v)
+	assert.Equal(t, 1, v.(int))
+
+	c.Delete(field, "my_config1")
+	c.Delete(field, "my_config2") // remove a non-exist key
+
+	v, _ = c.Get(field, "my_config1")
+	assert.Nil(t, v)
+
+	vInt, err := GetWithContextCache(ctx, field, "my_config1", func(context.Context, string) (int, error) {
+		return 1, nil
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, vInt)
+
+	v, _ = c.Get(field, "my_config1")
+	assert.EqualValues(t, 1, v)
+
+	defer test.MockVariableValue(&timeNow, func() time.Time {
+		return time.Now().Add(5 * time.Minute)
+	})()
+	v, _ = c.Get(field, "my_config1")
+	assert.Nil(t, v)
+}

modules/cache/ephemeral.go

@@ -0,0 +1,90 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cache
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/util"
+)
+
+// EphemeralCache is a cache that can be used to store data in a request level context
+// This is useful for caching data that is expensive to calculate and is likely to be
+// used multiple times in a request.
+type EphemeralCache struct {
+	data          map[any]map[any]any
+	lock          sync.RWMutex
+	created       time.Time
+	checkLifeTime time.Duration
+}
+
+var timeNow = time.Now
+
+func NewEphemeralCache(checkLifeTime ...time.Duration) *EphemeralCache {
+	return &EphemeralCache{
+		data:          make(map[any]map[any]any),
+		created:       timeNow(),
+		checkLifeTime: util.OptionalArg(checkLifeTime, 0),
+	}
+}
+
+func (cc *EphemeralCache) checkExceededLifeTime(tp, key any) bool {
+	if cc.checkLifeTime > 0 && timeNow().Sub(cc.created) > cc.checkLifeTime {
+		log.Warn("EphemeralCache is expired, is highly likely to be abused for long-life tasks: %v, %v", tp, key)
+		return true
+	}
+	return false
+}
+
+func (cc *EphemeralCache) Get(tp, key any) (any, bool) {
+	if cc.checkExceededLifeTime(tp, key) {
+		return nil, false
+	}
+	cc.lock.RLock()
+	defer cc.lock.RUnlock()
+	ret, ok := cc.data[tp][key]
+	return ret, ok
+}
+
+func (cc *EphemeralCache) Put(tp, key, value any) {
+	if cc.checkExceededLifeTime(tp, key) {
+		return
+	}
+
+	cc.lock.Lock()
+	defer cc.lock.Unlock()
+
+	d := cc.data[tp]
+	if d == nil {
+		d = make(map[any]any)
+		cc.data[tp] = d
+	}
+	d[key] = value
+}
+
+func (cc *EphemeralCache) Delete(tp, key any) {
+	if cc.checkExceededLifeTime(tp, key) {
+		return
+	}
+
+	cc.lock.Lock()
+	defer cc.lock.Unlock()
+	delete(cc.data[tp], key)
+}
+
+func GetWithEphemeralCache[T, K any](ctx context.Context, c *EphemeralCache, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) {
+	v, has := c.Get(groupKey, targetKey)
+	if vv, ok := v.(T); has && ok {
+		return vv, nil
+	}
+	t, err := f(ctx, targetKey)
+	if err != nil {
+		return t, err
+	}
+	c.Put(groupKey, targetKey, t)
+	return t, nil
+}

modules/cache/string_cache.go

@@ -0,0 +1,120 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cache
+
+import (
+	"errors"
+	"strings"
+
+	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+
+	chi_cache "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here
+)
+
+type GetJSONError struct {
+	err         error
+	cachedError string // Golang error can't be stored in cache, only the string message could be stored
+}
+
+func (e *GetJSONError) ToError() error {
+	if e.err != nil {
+		return e.err
+	}
+	return errors.New("cached error: " + e.cachedError)
+}
+
+type StringCache interface {
+	Ping() error
+
+	Get(key string) (string, bool)
+	Put(key, value string, ttl int64) error
+	Delete(key string) error
+	IsExist(key string) bool
+
+	PutJSON(key string, v any, ttl int64) error
+	GetJSON(key string, ptr any) (exist bool, err *GetJSONError)
+
+	ChiCache() chi_cache.Cache
+}
+
+type stringCache struct {
+	chiCache chi_cache.Cache
+}
+
+func NewStringCache(cacheConfig setting.Cache) (StringCache, error) {
+	adapter := util.IfZero(cacheConfig.Adapter, "memory")
+	interval := util.IfZero(cacheConfig.Interval, 60)
+	cc, err := chi_cache.NewCacher(chi_cache.Options{
+		Adapter:       adapter,
+		AdapterConfig: cacheConfig.Conn,
+		Interval:      interval,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &stringCache{chiCache: cc}, nil
+}
+
+func (sc *stringCache) Ping() error {
+	return sc.chiCache.Ping()
+}
+
+func (sc *stringCache) Get(key string) (string, bool) {
+	v := sc.chiCache.Get(key)
+	if v == nil {
+		return "", false
+	}
+	s, ok := v.(string)
+	return s, ok
+}
+
+func (sc *stringCache) Put(key, value string, ttl int64) error {
+	return sc.chiCache.Put(key, value, ttl)
+}
+
+func (sc *stringCache) Delete(key string) error {
+	return sc.chiCache.Delete(key)
+}
+
+func (sc *stringCache) IsExist(key string) bool {
+	return sc.chiCache.IsExist(key)
+}
+
+const cachedErrorPrefix = "<CACHED-ERROR>:"
+
+func (sc *stringCache) PutJSON(key string, v any, ttl int64) error {
+	var s string
+	switch v := v.(type) {
+	case error:
+		s = cachedErrorPrefix + v.Error()
+	default:
+		b, err := json.Marshal(v)
+		if err != nil {
+			return err
+		}
+		s = util.UnsafeBytesToString(b)
+	}
+	return sc.chiCache.Put(key, s, ttl)
+}
+
+func (sc *stringCache) GetJSON(key string, ptr any) (exist bool, getErr *GetJSONError) {
+	s, ok := sc.Get(key)
+	if !ok || s == "" {
+		return false, nil
+	}
+	s, isCachedError := strings.CutPrefix(s, cachedErrorPrefix)
+	if isCachedError {
+		return true, &GetJSONError{cachedError: s}
+	}
+	if err := json.Unmarshal(util.UnsafeStringToBytes(s), ptr); err != nil {
+		return false, &GetJSONError{err: err}
+	}
+	return true, nil
+}
+
+func (sc *stringCache) ChiCache() chi_cache.Cache {
+	return sc.chiCache
+}

modules/cachegroup/cachegroup.go

@@ -0,0 +1,12 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cachegroup
+
+const (
+	User               = "user"
+	EmailAvatarLink    = "email_avatar_link"
+	UserEmailAddresses = "user_email_addresses"
+	GPGKeyWithSubKeys  = "gpg_key_with_subkeys"
+	RepoUserPermission = "repo_user_permission"
+)

modules/charset/ambiguous.go

@@ -0,0 +1,59 @@
+// This file is generated by modules/charset/ambiguous/generate.go DO NOT EDIT
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package charset
+
+import (
+	"sort"
+	"strings"
+	"unicode"
+
+	"code.gitea.io/gitea/modules/translation"
+)
+
+// AmbiguousTablesForLocale provides the table of ambiguous characters for this locale.
+func AmbiguousTablesForLocale(locale translation.Locale) []*AmbiguousTable {
+	key := locale.Language()
+	var table *AmbiguousTable
+	var ok bool
+	for len(key) > 0 {
+		if table, ok = AmbiguousCharacters[key]; ok {
+			break
+		}
+		idx := strings.LastIndexAny(key, "-_")
+		if idx < 0 {
+			key = ""
+		} else {
+			key = key[:idx]
+		}
+	}
+	if table == nil && (locale.Language() == "zh-CN" || locale.Language() == "zh_CN") {
+		table = AmbiguousCharacters["zh-hans"]
+	}
+	if table == nil && strings.HasPrefix(locale.Language(), "zh") {
+		table = AmbiguousCharacters["zh-hant"]
+	}
+	if table == nil {
+		table = AmbiguousCharacters["_default"]
+	}
+
+	return []*AmbiguousTable{
+		table,
+		AmbiguousCharacters["_common"],
+	}
+}
+
+func isAmbiguous(r rune, confusableTo *rune, tables ...*AmbiguousTable) bool {
+	for _, table := range tables {
+		if !unicode.Is(table.RangeTable, r) {
+			continue
+		}
+		i := sort.Search(len(table.Confusable), func(i int) bool {
+			return table.Confusable[i] >= r
+		})
+		(*confusableTo) = table.With[i]
+		return true
+	}
+	return false
+}

modules/charset/ambiguous/generate.go

@@ -0,0 +1,188 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/format"
+	"os"
+	"sort"
+	"text/template"
+	"unicode"
+
+	"code.gitea.io/gitea/modules/json"
+
+	"golang.org/x/text/unicode/rangetable"
+)
+
+// ambiguous.json provides a one to one mapping of ambiguous characters to other characters
+// See https://github.com/hediet/vscode-unicode-data/blob/main/out/ambiguous.json
+
+type AmbiguousTable struct {
+	Confusable []rune
+	With       []rune
+	Locale     string
+	RangeTable *unicode.RangeTable
+}
+
+type RunePair struct {
+	Confusable rune
+	With       rune
+}
+
+var verbose bool
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, `%s: Generate AmbiguousCharacter
+
+Usage: %[1]s [-v] [-o output.go] ambiguous.json
+`, os.Args[0])
+		flag.PrintDefaults()
+	}
+
+	output := ""
+	flag.BoolVar(&verbose, "v", false, "verbose output")
+	flag.StringVar(&output, "o", "ambiguous_gen.go", "file to output to")
+	flag.Parse()
+	input := flag.Arg(0)
+	if input == "" {
+		input = "ambiguous.json"
+	}
+
+	bs, err := os.ReadFile(input)
+	if err != nil {
+		fatalf("Unable to read: %s Err: %v", input, err)
+	}
+
+	var unwrapped string
+	if err := json.Unmarshal(bs, &unwrapped); err != nil {
+		fatalf("Unable to unwrap content in: %s Err: %v", input, err)
+	}
+
+	fromJSON := map[string][]uint32{}
+	if err := json.Unmarshal([]byte(unwrapped), &fromJSON); err != nil {
+		fatalf("Unable to unmarshal content in: %s Err: %v", input, err)
+	}
+
+	tables := make([]*AmbiguousTable, 0, len(fromJSON))
+	for locale, chars := range fromJSON {
+		table := &AmbiguousTable{Locale: locale}
+		table.Confusable = make([]rune, 0, len(chars)/2)
+		table.With = make([]rune, 0, len(chars)/2)
+		pairs := make([]RunePair, len(chars)/2)
+		for i := 0; i < len(chars); i += 2 {
+			pairs[i/2].Confusable, pairs[i/2].With = rune(chars[i]), rune(chars[i+1])
+		}
+		sort.Slice(pairs, func(i, j int) bool {
+			return pairs[i].Confusable < pairs[j].Confusable
+		})
+		for _, pair := range pairs {
+			table.Confusable = append(table.Confusable, pair.Confusable)
+			table.With = append(table.With, pair.With)
+		}
+		table.RangeTable = rangetable.New(table.Confusable...)
+		tables = append(tables, table)
+	}
+	sort.Slice(tables, func(i, j int) bool {
+		return tables[i].Locale < tables[j].Locale
+	})
+	data := map[string]any{
+		"Tables": tables,
+	}
+
+	if err := runTemplate(generatorTemplate, output, &data); err != nil {
+		fatalf("Unable to run template: %v", err)
+	}
+}
+
+func runTemplate(t *template.Template, filename string, data any) error {
+	buf := bytes.NewBuffer(nil)
+	if err := t.Execute(buf, data); err != nil {
+		return fmt.Errorf("unable to execute template: %w", err)
+	}
+	bs, err := format.Source(buf.Bytes())
+	if err != nil {
+		verbosef("Bad source:\n%s", buf.String())
+		return fmt.Errorf("unable to format source: %w", err)
+	}
+
+	old, err := os.ReadFile(filename)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to read old file %s because %w", filename, err)
+	} else if err == nil {
+		if bytes.Equal(bs, old) {
+			// files are the same don't rewrite it.
+			return nil
+		}
+	}
+
+	file, err := os.Create(filename)
+	if err != nil {
+		return fmt.Errorf("failed to create file %s because %w", filename, err)
+	}
+	defer file.Close()
+	_, err = file.Write(bs)
+	if err != nil {
+		return fmt.Errorf("unable to write generated source: %w", err)
+	}
+	return nil
+}
+
+var generatorTemplate = template.Must(template.New("ambiguousTemplate").Parse(`// This file is generated by modules/charset/ambiguous/generate.go DO NOT EDIT
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+
+package charset
+
+import "unicode"
+
+// This file is generated from https://github.com/hediet/vscode-unicode-data/blob/main/out/ambiguous.json
+
+// AmbiguousTable matches a confusable rune with its partner for the Locale
+type AmbiguousTable struct {
+	Confusable []rune
+	With       []rune
+	Locale     string
+	RangeTable *unicode.RangeTable
+}
+
+// AmbiguousCharacters provides a map by locale name to the confusable characters in that locale
+var AmbiguousCharacters = map[string]*AmbiguousTable{
+	{{range .Tables}}{{printf "%q:" .Locale}} {
+			Confusable: []rune{ {{range .Confusable}}{{.}},{{end}} },
+			With: []rune{ {{range .With}}{{.}},{{end}} },
+			Locale: {{printf "%q" .Locale}},
+			RangeTable: &unicode.RangeTable{
+				R16: []unicode.Range16{
+			{{range .RangeTable.R16 }}		{Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}},
+			{{end}}	},
+				R32: []unicode.Range32{
+			{{range .RangeTable.R32}}		{Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}},
+			{{end}}	},
+				LatinOffset: {{.RangeTable.LatinOffset}},
+			},
+		},
+	{{end}}
+}
+
+`))
+
+func logf(format string, args ...any) {
+	fmt.Fprintf(os.Stderr, format+"\n", args...)
+}
+
+func verbosef(format string, args ...any) {
+	if verbose {
+		logf(format, args...)
+	}
+}
+
+func fatalf(format string, args ...any) {
+	logf("fatal: "+format+"\n", args...)
+	os.Exit(1)
+}

modules/charset/ambiguous_gen_test.go

@@ -0,0 +1,31 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package charset
+
+import (
+	"sort"
+	"testing"
+	"unicode"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAmbiguousCharacters(t *testing.T) {
+	for locale, ambiguous := range AmbiguousCharacters {
+		assert.Equal(t, locale, ambiguous.Locale)
+		assert.Len(t, ambiguous.With, len(ambiguous.Confusable))
+		assert.True(t, sort.SliceIsSorted(ambiguous.Confusable, func(i, j int) bool {
+			return ambiguous.Confusable[i] < ambiguous.Confusable[j]
+		}))
+
+		for _, confusable := range ambiguous.Confusable {
+			assert.True(t, unicode.Is(ambiguous.RangeTable, confusable))
+			i := sort.Search(len(ambiguous.Confusable), func(j int) bool {
+				return ambiguous.Confusable[j] >= confusable
+			})
+			found := i < len(ambiguous.Confusable) && ambiguous.Confusable[i] == confusable
+			assert.True(t, found, "%c is not in %d", confusable, i)
+		}
+	}
+}

modules/charset/breakwriter.go

@@ -0,0 +1,43 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package charset
+
+import (
+	"bytes"
+	"io"
+)
+
+// BreakWriter wraps an io.Writer to always write '\n' as '<br>'
+type BreakWriter struct {
+	io.Writer
+}
+
+// Write writes the provided byte slice transparently replacing '\n' with '<br>'
+func (b *BreakWriter) Write(bs []byte) (n int, err error) {
+	pos := 0
+	for pos < len(bs) {
+		idx := bytes.IndexByte(bs[pos:], '\n')
+		if idx < 0 {
+			wn, err := b.Writer.Write(bs[pos:])
+			return n + wn, err
+		}
+
+		if idx > 0 {
+			wn, err := b.Writer.Write(bs[pos : pos+idx])
+			n += wn
+			if err != nil {
+				return n, err
+			}
+		}
+
+		if _, err = b.Writer.Write([]byte("<br>")); err != nil {
+			return n, err
+		}
+		pos += idx + 1
+
+		n++
+	}
+
+	return n, err
+}

modules/charset/breakwriter_test.go

@@ -0,0 +1,68 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package charset
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestBreakWriter_Write(t *testing.T) {
+	tests := []struct {
+		name    string
+		kase    string
+		expect  string
+		wantErr bool
+	}{
+		{
+			name:   "noline",
+			kase:   "abcdefghijklmnopqrstuvwxyz",
+			expect: "abcdefghijklmnopqrstuvwxyz",
+		},
+		{
+			name:   "endline",
+			kase:   "abcdefghijklmnopqrstuvwxyz\n",
+			expect: "abcdefghijklmnopqrstuvwxyz<br>",
+		},
+		{
+			name:   "startline",
+			kase:   "\nabcdefghijklmnopqrstuvwxyz",
+			expect: "<br>abcdefghijklmnopqrstuvwxyz",
+		},
+		{
+			name:   "onlyline",
+			kase:   "\n\n\n",
+			expect: "<br><br><br>",
+		},
+		{
+			name:   "empty",
+			kase:   "",
+			expect: "",
+		},
+		{
+			name:   "midline",
+			kase:   "\nabc\ndefghijkl\nmnopqrstuvwxy\nz",
+			expect: "<br>abc<br>defghijkl<br>mnopqrstuvwxy<br>z",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			buf := &strings.Builder{}
+			b := &BreakWriter{
+				Writer: buf,
+			}
+			n, err := b.Write([]byte(tt.kase))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("BreakWriter.Write() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if n != len(tt.kase) {
+				t.Errorf("BreakWriter.Write() = %v, want %v", n, len(tt.kase))
+			}
+			if buf.String() != tt.expect {
+				t.Errorf("BreakWriter.Write() wrote %q, want %v", buf.String(), tt.expect)
+			}
+		})
+	}
+}

modules/charset/charset.go

@@ -0,0 +1,211 @@
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package charset
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"unicode/utf8"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+
+	"github.com/gogs/chardet"
+	"golang.org/x/net/html/charset"
+	"golang.org/x/text/transform"
+)
+
+// UTF8BOM is the utf-8 byte-order marker
+var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
+
+type ConvertOpts struct {
+	KeepBOM bool
+}
+
+// ToUTF8WithFallbackReader detects the encoding of content and converts to UTF-8 reader if possible
+func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader {
+	buf := make([]byte, 2048)
+	n, err := util.ReadAtMost(rd, buf)
+	if err != nil {
+		return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
+	}
+
+	charsetLabel, err := DetectEncoding(buf[:n])
+	if err != nil || charsetLabel == "UTF-8" {
+		return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return io.MultiReader(bytes.NewReader(buf[:n]), rd)
+	}
+
+	return transform.NewReader(
+		io.MultiReader(
+			bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)),
+			rd,
+		),
+		encoding.NewDecoder(),
+	)
+}
+
+// ToUTF8 converts content to UTF8 encoding
+func ToUTF8(content []byte, opts ConvertOpts) (string, error) {
+	charsetLabel, err := DetectEncoding(content)
+	if err != nil {
+		return "", err
+	} else if charsetLabel == "UTF-8" {
+		return string(MaybeRemoveBOM(content, opts)), nil
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel)
+	}
+
+	// If there is an error, we concatenate the nicely decoded part and the
+	// original left over. This way we won't lose much data.
+	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
+	if err != nil {
+		result = append(result, content[n:]...)
+	}
+
+	result = MaybeRemoveBOM(result, opts)
+
+	return string(result), err
+}
+
+// ToUTF8WithFallback detects the encoding of content and converts to UTF-8 if possible
+func ToUTF8WithFallback(content []byte, opts ConvertOpts) []byte {
+	bs, _ := io.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content), opts))
+	return bs
+}
+
+// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible
+func ToUTF8DropErrors(content []byte, opts ConvertOpts) []byte {
+	charsetLabel, err := DetectEncoding(content)
+	if err != nil || charsetLabel == "UTF-8" {
+		return MaybeRemoveBOM(content, opts)
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return content
+	}
+
+	// We ignore any non-decodable parts from the file.
+	// Some parts might be lost
+	var decoded []byte
+	decoder := encoding.NewDecoder()
+	idx := 0
+	for {
+		result, n, err := transform.Bytes(decoder, content[idx:])
+		decoded = append(decoded, result...)
+		if err == nil {
+			break
+		}
+		decoded = append(decoded, ' ')
+		idx = idx + n + 1
+		if idx >= len(content) {
+			break
+		}
+	}
+
+	return MaybeRemoveBOM(decoded, opts)
+}
+
+// MaybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false
+func MaybeRemoveBOM(content []byte, opts ConvertOpts) []byte {
+	if opts.KeepBOM {
+		return content
+	}
+	if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
+		return content[3:]
+	}
+	return content
+}
+
+// DetectEncoding detect the encoding of content
+func DetectEncoding(content []byte) (string, error) {
+	// First we check if the content represents valid utf8 content excepting a truncated character at the end.
+
+	// Now we could decode all the runes in turn but this is not necessarily the cheapest thing to do
+	// instead we walk backwards from the end to trim off a the incomplete character
+	toValidate := content
+	end := len(toValidate) - 1
+
+	if end < 0 {
+		// no-op
+	} else if toValidate[end]>>5 == 0b110 {
+		// Incomplete 1 byte extension e.g. © <c2><a9> which has been truncated to <c2>
+		toValidate = toValidate[:end]
+	} else if end > 0 && toValidate[end]>>6 == 0b10 && toValidate[end-1]>>4 == 0b1110 {
+		// Incomplete 2 byte extension e.g. ⛔ <e2><9b><94> which has been truncated to <e2><9b>
+		toValidate = toValidate[:end-1]
+	} else if end > 1 && toValidate[end]>>6 == 0b10 && toValidate[end-1]>>6 == 0b10 && toValidate[end-2]>>3 == 0b11110 {
+		// Incomplete 3 byte extension e.g. 💩 <f0><9f><92><a9> which has been truncated to <f0><9f><92>
+		toValidate = toValidate[:end-2]
+	}
+	if utf8.Valid(toValidate) {
+		log.Debug("Detected encoding: utf-8 (fast)")
+		return "UTF-8", nil
+	}
+
+	textDetector := chardet.NewTextDetector()
+	var detectContent []byte
+	if len(content) < 1024 {
+		// Check if original content is valid
+		if _, err := textDetector.DetectBest(content); err != nil {
+			return "", err
+		}
+		times := 1024 / len(content)
+		detectContent = make([]byte, 0, times*len(content))
+		for range times {
+			detectContent = append(detectContent, content...)
+		}
+	} else {
+		detectContent = content
+	}
+
+	// Now we can't use DetectBest or just results[0] because the result isn't stable - so we need a tie break
+	results, err := textDetector.DetectAll(detectContent)
+	if err != nil {
+		if err == chardet.NotDetectedError && len(setting.Repository.AnsiCharset) > 0 {
+			log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
+			return setting.Repository.AnsiCharset, nil
+		}
+		return "", err
+	}
+
+	topConfidence := results[0].Confidence
+	topResult := results[0]
+	priority, has := setting.Repository.DetectedCharsetScore[strings.ToLower(strings.TrimSpace(topResult.Charset))]
+	for _, result := range results {
+		// As results are sorted in confidence order - if we have a different confidence
+		// we know it's less than the current confidence and can break out of the loop early
+		if result.Confidence != topConfidence {
+			break
+		}
+
+		// Otherwise check if this results is earlier in the DetectedCharsetOrder than our current top guess
+		resultPriority, resultHas := setting.Repository.DetectedCharsetScore[strings.ToLower(strings.TrimSpace(result.Charset))]
+		if resultHas && (!has || resultPriority < priority) {
+			topResult = result
+			priority = resultPriority
+			has = true
+		}
+	}
+
+	// FIXME: to properly decouple this function the fallback ANSI charset should be passed as an argument
+	if topResult.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
+		log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
+		return setting.Repository.AnsiCharset, err
+	}
+
+	log.Debug("Detected encoding: %s", topResult.Charset)
+	return topResult.Charset, err
+}

modules/charset/charset_test.go

@@ -0,0 +1,382 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package charset
+
+import (
+	"bytes"
+	"io"
+	"strings"
+	"testing"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func resetDefaultCharsetsOrder() {
+	defaultDetectedCharsetsOrder := make([]string, 0, len(setting.Repository.DetectedCharsetsOrder))
+	for _, charset := range setting.Repository.DetectedCharsetsOrder {
+		defaultDetectedCharsetsOrder = append(defaultDetectedCharsetsOrder, strings.ToLower(strings.TrimSpace(charset)))
+	}
+	setting.Repository.DetectedCharsetScore = map[string]int{}
+	i := 0
+	for _, charset := range defaultDetectedCharsetsOrder {
+		canonicalCharset := strings.ToLower(strings.TrimSpace(charset))
+		if _, has := setting.Repository.DetectedCharsetScore[canonicalCharset]; !has {
+			setting.Repository.DetectedCharsetScore[canonicalCharset] = i
+			i++
+		}
+	}
+}
+
+func TestMaybeRemoveBOM(t *testing.T) {
+	res := MaybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+
+	res = MaybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+}
+
+func TestToUTF8(t *testing.T) {
+	resetDefaultCharsetsOrder()
+
+	// Note: golang compiler seems so behave differently depending on the current
+	// locale, so some conversions might behave differently. For that reason, we don't
+	// depend on particular conversions but in expected behaviors.
+
+	res, err := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
+	assert.NoError(t, err)
+	assert.Equal(t, "ABC", res)
+
+	// "áéíóú"
+	res, err = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
+	assert.NoError(t, err)
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
+
+	// "áéíóú"
+	res, err = ToUTF8([]byte{
+		0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
+		0xc3, 0xba,
+	}, ConvertOpts{})
+	assert.NoError(t, err)
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
+
+	res, err = ToUTF8([]byte{
+		0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
+	}, ConvertOpts{})
+	assert.NoError(t, err)
+	stringMustStartWith(t, "Hola,", res)
+	stringMustEndWith(t, "AAA.", res)
+
+	res, err = ToUTF8([]byte{
+		0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
+	}, ConvertOpts{})
+	assert.NoError(t, err)
+	stringMustStartWith(t, "Hola,", res)
+	stringMustEndWith(t, "AAA.", res)
+
+	res, err = ToUTF8([]byte{
+		0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
+	}, ConvertOpts{})
+	assert.NoError(t, err)
+	stringMustStartWith(t, "Hola,", res)
+	stringMustEndWith(t, "AAA.", res)
+
+	// Japanese (Shift-JIS)
+	// 日属秘ぞしちゅ。
+	res, err = ToUTF8([]byte{
+		0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
+		0xBF, 0x82, 0xE3, 0x81, 0x42,
+	}, ConvertOpts{})
+	assert.NoError(t, err)
+	assert.Equal(t, []byte{
+		0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
+		0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
+	},
+		[]byte(res))
+
+	res, err = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
+	assert.NoError(t, err)
+	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
+}
+
+func TestToUTF8WithFallback(t *testing.T) {
+	resetDefaultCharsetsOrder()
+	// "ABC"
+	res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
+	assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
+
+	// "áéíóú"
+	res = ToUTF8WithFallback([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+
+	// UTF8 BOM + "áéíóú"
+	res = ToUTF8WithFallback([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+
+	// "Hola, así cómo ños"
+	res = ToUTF8WithFallback([]byte{
+		0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73,
+	}, ConvertOpts{})
+	assert.Equal(t, []byte{
+		0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63,
+		0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73,
+	}, res)
+
+	// "Hola, así cómo "
+	minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
+
+	res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}, ConvertOpts{})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+	res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}, ConvertOpts{})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+	// Japanese (Shift-JIS)
+	// "日属秘ぞしちゅ。"
+	res = ToUTF8WithFallback([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}, ConvertOpts{})
+	assert.Equal(t, []byte{
+		0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
+		0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
+	}, res)
+
+	res = ToUTF8WithFallback([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
+	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
+}
+
+func TestToUTF8DropErrors(t *testing.T) {
+	resetDefaultCharsetsOrder()
+	// "ABC"
+	res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
+	assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
+
+	// "áéíóú"
+	res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+
+	// UTF8 BOM + "áéíóú"
+	res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+
+	// "Hola, así cómo ños"
+	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}, ConvertOpts{})
+	assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73}, res[:8])
+	assert.Equal(t, []byte{0x73}, res[len(res)-1:])
+
+	// "Hola, así cómo "
+	minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
+
+	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}, ConvertOpts{})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}, ConvertOpts{})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+	// Japanese (Shift-JIS)
+	// "日属秘ぞしちゅ。"
+	res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}, ConvertOpts{})
+	assert.Equal(t, []byte{
+		0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
+		0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
+	}, res)
+
+	res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
+	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
+}
+
+func TestDetectEncoding(t *testing.T) {
+	resetDefaultCharsetsOrder()
+	testSuccess := func(b []byte, expected string) {
+		encoding, err := DetectEncoding(b)
+		assert.NoError(t, err)
+		assert.Equal(t, expected, encoding)
+	}
+	// utf-8
+	b := []byte("just some ascii")
+	testSuccess(b, "UTF-8")
+
+	// utf-8-sig: "hey" (with BOM)
+	b = []byte{0xef, 0xbb, 0xbf, 0x68, 0x65, 0x79}
+	testSuccess(b, "UTF-8")
+
+	// utf-16: "hey<accented G>"
+	b = []byte{0xff, 0xfe, 0x68, 0x00, 0x65, 0x00, 0x79, 0x00, 0xf4, 0x01}
+	testSuccess(b, "UTF-16LE")
+
+	// iso-8859-1: d<accented e>cor<newline>
+	b = []byte{0x44, 0xe9, 0x63, 0x6f, 0x72, 0x0a}
+	encoding, err := DetectEncoding(b)
+	assert.NoError(t, err)
+	assert.Contains(t, encoding, "ISO-8859-1")
+
+	old := setting.Repository.AnsiCharset
+	setting.Repository.AnsiCharset = "placeholder"
+	defer func() {
+		setting.Repository.AnsiCharset = old
+	}()
+	testSuccess(b, "placeholder")
+
+	// invalid bytes
+	b = []byte{0xfa}
+	_, err = DetectEncoding(b)
+	assert.Error(t, err)
+}
+
+func stringMustStartWith(t *testing.T, expected, value string) {
+	assert.Equal(t, expected, value[:len(expected)])
+}
+
+func stringMustEndWith(t *testing.T, expected, value string) {
+	assert.Equal(t, expected, value[len(value)-len(expected):])
+}
+
+func TestToUTF8WithFallbackReader(t *testing.T) {
+	resetDefaultCharsetsOrder()
+
+	for testLen := range 2048 {
+		pattern := "    test { () }\n"
+		input := ""
+		for len(input) < testLen {
+			input += pattern
+		}
+		input = input[:testLen]
+		input += "// Выключаем"
+		rd := ToUTF8WithFallbackReader(bytes.NewReader([]byte(input)), ConvertOpts{})
+		r, _ := io.ReadAll(rd)
+		assert.Equalf(t, input, string(r), "testing string len=%d", testLen)
+	}
+
+	truncatedOneByteExtension := failFastBytes
+	encoding, _ := DetectEncoding(truncatedOneByteExtension)
+	assert.Equal(t, "UTF-8", encoding)
+
+	truncatedTwoByteExtension := failFastBytes
+	truncatedTwoByteExtension[len(failFastBytes)-1] = 0x9b
+	truncatedTwoByteExtension[len(failFastBytes)-2] = 0xe2
+
+	encoding, _ = DetectEncoding(truncatedTwoByteExtension)
+	assert.Equal(t, "UTF-8", encoding)
+
+	truncatedThreeByteExtension := failFastBytes
+	truncatedThreeByteExtension[len(failFastBytes)-1] = 0x92
+	truncatedThreeByteExtension[len(failFastBytes)-2] = 0x9f
+	truncatedThreeByteExtension[len(failFastBytes)-3] = 0xf0
+
+	encoding, _ = DetectEncoding(truncatedThreeByteExtension)
+	assert.Equal(t, "UTF-8", encoding)
+}
+
+var failFastBytes = []byte{
+	0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x74, 0x6f,
+	0x6f, 0x6c, 0x73, 0x2e, 0x61, 0x6e, 0x74, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x64, 0x65, 0x66, 0x73, 0x2e, 0x63, 0x6f, 0x6e,
+	0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4f, 0x73, 0x0a, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6f, 0x72, 0x67,
+	0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f,
+	0x74, 0x2e, 0x67, 0x72, 0x61, 0x64, 0x6c, 0x65, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x2e, 0x72, 0x75, 0x6e, 0x2e, 0x42,
+	0x6f, 0x6f, 0x74, 0x52, 0x75, 0x6e, 0x0a, 0x0a, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x20, 0x7b, 0x0a, 0x20, 0x20,
+	0x20, 0x20, 0x69, 0x64, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d,
+	0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x22, 0x29, 0x0a, 0x7d, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x65,
+	0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65,
+	0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a,
+	0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x61, 0x70, 0x69, 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d,
+	0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74,
+	0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x61, 0x70, 0x69, 0x2d, 0x64, 0x6f, 0x63, 0x73, 0x22, 0x29,
+	0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x64, 0x62,
+	0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69,
+	0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a,
+	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65,
+	0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a,
+	0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2d, 0x66,
+	0x73, 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x3a, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2d, 0x6d, 0x71, 0x22, 0x29, 0x29, 0x0a, 0x0a,
+	0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22,
+	0x6a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e,
+	0x2d, 0x61, 0x75, 0x74, 0x68, 0x2d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2d, 0x73, 0x74, 0x61, 0x72, 0x74,
+	0x65, 0x72, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63,
+	0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2d, 0x68, 0x61, 0x6c, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c,
+	0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e,
+	0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2d, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x29, 0x0a,
+	0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28,
+	0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b,
+	0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x2d, 0x73, 0x74,
+	0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x77, 0x65, 0x62, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c,
+	0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69,
+	0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x3a, 0x73, 0x70, 0x72,
+	0x69, 0x6e, 0x67, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x2d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x61, 0x6f, 0x70,
+	0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f,
+	0x6e, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f,
+	0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x2d,
+	0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x61, 0x63, 0x74, 0x75, 0x61, 0x74, 0x6f, 0x72, 0x22, 0x29, 0x0a, 0x20,
+	0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f,
+	0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x63,
+	0x6c, 0x6f, 0x75, 0x64, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x73, 0x74,
+	0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x22, 0x29, 0x0a, 0x20, 0x20,
+	0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72,
+	0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x63, 0x6c,
+	0x6f, 0x75, 0x64, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x73, 0x74, 0x61,
+	0x72, 0x74, 0x65, 0x72, 0x2d, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2d, 0x61, 0x6c, 0x6c, 0x22, 0x29, 0x0a, 0x20, 0x20,
+	0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72,
+	0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x63, 0x6c,
+	0x6f, 0x75, 0x64, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x73, 0x74, 0x61,
+	0x72, 0x74, 0x65, 0x72, 0x2d, 0x73, 0x6c, 0x65, 0x75, 0x74, 0x68, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d,
+	0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70,
+	0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x72, 0x65, 0x74, 0x72, 0x79, 0x3a,
+	0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x72, 0x65, 0x74, 0x72, 0x79, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20,
+	0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x63, 0x68, 0x2e, 0x71,
+	0x6f, 0x73, 0x2e, 0x6c, 0x6f, 0x67, 0x62, 0x61, 0x63, 0x6b, 0x3a, 0x6c, 0x6f, 0x67, 0x62, 0x61, 0x63, 0x6b, 0x2d, 0x63,
+	0x6c, 0x61, 0x73, 0x73, 0x69, 0x63, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d,
+	0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x69, 0x6f, 0x2e, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x6d, 0x65,
+	0x74, 0x65, 0x72, 0x3a, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x2d, 0x72, 0x65, 0x67, 0x69, 0x73,
+	0x74, 0x72, 0x79, 0x2d, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20,
+	0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x6b, 0x6f, 0x74,
+	0x6c, 0x69, 0x6e, 0x28, 0x22, 0x73, 0x74, 0x64, 0x6c, 0x69, 0x62, 0x22, 0x29, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20,
+	0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f,
+	0x2f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64,
+	0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x2e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f,
+	0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74,
+	0x65, 0x73, 0x74, 0x49, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6a,
+	0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2d,
+	0x74, 0x65, 0x73, 0x74, 0x22, 0x29, 0x0a, 0x7d, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x20, 0x70, 0x61, 0x74, 0x63, 0x68, 0x4a,
+	0x61, 0x72, 0x20, 0x62, 0x79, 0x20, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72,
+	0x69, 0x6e, 0x67, 0x28, 0x4a, 0x61, 0x72, 0x3a, 0x3a, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20,
+	0x20, 0x20, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x69, 0x66, 0x69, 0x65, 0x72, 0x2e,
+	0x73, 0x65, 0x74, 0x28, 0x22, 0x70, 0x61, 0x74, 0x63, 0x68, 0x65, 0x64, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20,
+	0x76, 0x61, 0x6c, 0x20, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x70, 0x61, 0x74, 0x68,
+	0x20, 0x62, 0x79, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x67,
+	0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74,
+	0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65,
+	0x73, 0x28, 0x22, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x2d, 0x50, 0x61, 0x74, 0x68, 0x22, 0x20, 0x74, 0x6f, 0x20, 0x6f, 0x62,
+	0x6a, 0x65, 0x63, 0x74, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70,
+	0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x20, 0x76, 0x61, 0x6c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x3d,
+	0x20, 0x22, 0x66, 0x69, 0x6c, 0x65, 0x3a, 0x2f, 0x2b, 0x22, 0x2e, 0x74, 0x6f, 0x52, 0x65, 0x67, 0x65, 0x78, 0x28, 0x29,
+	0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64,
+	0x65, 0x20, 0x66, 0x75, 0x6e, 0x20, 0x74, 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x28, 0x29, 0x3a, 0x20, 0x53, 0x74,
+	0x72, 0x69, 0x6e, 0x67, 0x20, 0x3d, 0x20, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x70,
+	0x61, 0x74, 0x68, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x54, 0x6f, 0x53, 0x74, 0x72, 0x69,
+	0x6e, 0x67, 0x28, 0x22, 0x20, 0x22, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+	0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x74, 0x2e, 0x74, 0x6f, 0x55, 0x52, 0x49, 0x28, 0x29, 0x2e, 0x74, 0x6f, 0x55,
+	0x52, 0x4c, 0x28, 0x29, 0x2e, 0x74, 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x28, 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c,
+	0x61, 0x63, 0x65, 0x46, 0x69, 0x72, 0x73, 0x74, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2c, 0x20, 0x22, 0x2f,
+	0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20,
+	0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a, 0x0a, 0x74, 0x61, 0x73,
+	0x6b, 0x73, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x64, 0x3c, 0x42, 0x6f, 0x6f, 0x74, 0x52, 0x75, 0x6e, 0x3e, 0x28, 0x22, 0x62,
+	0x6f, 0x6f, 0x74, 0x52, 0x75, 0x6e, 0x22, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x4f,
+	0x73, 0x2e, 0x69, 0x73, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x28, 0x4f, 0x73, 0x2e, 0x46, 0x41, 0x4d, 0x49, 0x4c, 0x59,
+	0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x53, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+	0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x70, 0x61, 0x74, 0x68, 0x20, 0x3d, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x28, 0x73,
+	0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x74, 0x73, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x64, 0x28, 0x22, 0x6d, 0x61, 0x69,
+	0x6e, 0x22, 0x29, 0x2e, 0x6d, 0x61, 0x70, 0x20, 0x7b, 0x20, 0x69, 0x74, 0x2e, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x20,
+	0x7d, 0x2c, 0x20, 0x70, 0x61, 0x74, 0x63, 0x68, 0x4a, 0x61, 0x72, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a,
+	0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0xd0,
+}

modules/charset/escape.go

@@ -0,0 +1,44 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:generate go run invisible/generate.go -v -o ./invisible_gen.go
+
+//go:generate go run ambiguous/generate.go -v -o ./ambiguous_gen.go ambiguous/ambiguous.json
+
+package charset
+
+import (
+	"html/template"
+	"io"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/translation"
+)
+
+// RuneNBSP is the codepoint for NBSP
+const RuneNBSP = 0xa0
+
+// EscapeControlHTML escapes the unicode control sequences in a provided html document
+func EscapeControlHTML(html template.HTML, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output template.HTML) {
+	sb := &strings.Builder{}
+	escaped, _ = EscapeControlReader(strings.NewReader(string(html)), sb, locale, allowed...) // err has been handled in EscapeControlReader
+	return escaped, template.HTML(sb.String())
+}
+
+// EscapeControlReader escapes the unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus
+func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) {
+	if !setting.UI.AmbiguousUnicodeDetection {
+		_, err = io.Copy(writer, reader)
+		return &EscapeStatus{}, err
+	}
+	outputStream := &HTMLStreamerWriter{Writer: writer}
+	streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)
+
+	if err = StreamHTML(reader, streamer); err != nil {
+		streamer.escaped.HasError = true
+		log.Error("Error whilst escaping: %v", err)
+	}
+	return streamer.escaped, err
+}

modules/charset/escape_status.go

@@ -0,0 +1,27 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package charset
+
+// EscapeStatus represents the findings of the unicode escaper
+type EscapeStatus struct {
+	Escaped      bool
+	HasError     bool
+	HasBadRunes  bool
+	HasInvisible bool
+	HasAmbiguous bool
+}
+
+// Or combines two EscapeStatus structs into one representing the conjunction of the two
+func (status *EscapeStatus) Or(other *EscapeStatus) *EscapeStatus {
+	st := status
+	if status == nil {
+		st = &EscapeStatus{}
+	}
+	st.Escaped = st.Escaped || other.Escaped
+	st.HasError = st.HasError || other.HasError
+	st.HasBadRunes = st.HasBadRunes || other.HasBadRunes
+	st.HasAmbiguous = st.HasAmbiguous || other.HasAmbiguous
+	st.HasInvisible = st.HasInvisible || other.HasInvisible
+	return st
+}

modules/charset/escape_stream.go

@@ -0,0 +1,289 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package charset
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"code.gitea.io/gitea/modules/translation"
+
+	"golang.org/x/net/html"
+)
+
+// VScode defaultWordRegexp
+var defaultWordRegexp = regexp.MustCompile(`(-?\d*\.\d\w*)|([^\` + "`" + `\~\!\@\#\$\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s\x00-\x1f]+)`)
+
+func NewEscapeStreamer(locale translation.Locale, next HTMLStreamer, allowed ...rune) HTMLStreamer {
+	allowedM := make(map[rune]bool, len(allowed))
+	for _, v := range allowed {
+		allowedM[v] = true
+	}
+	return &escapeStreamer{
+		escaped:                 &EscapeStatus{},
+		PassthroughHTMLStreamer: *NewPassthroughStreamer(next),
+		locale:                  locale,
+		ambiguousTables:         AmbiguousTablesForLocale(locale),
+		allowed:                 allowedM,
+	}
+}
+
+type escapeStreamer struct {
+	PassthroughHTMLStreamer
+	escaped         *EscapeStatus
+	locale          translation.Locale
+	ambiguousTables []*AmbiguousTable
+	allowed         map[rune]bool
+}
+
+func (e *escapeStreamer) EscapeStatus() *EscapeStatus {
+	return e.escaped
+}
+
+// Text tells the next streamer there is a text
+func (e *escapeStreamer) Text(data string) error {
+	sb := &strings.Builder{}
+	var until int
+	var next int
+	pos := 0
+	if len(data) > len(UTF8BOM) && data[:len(UTF8BOM)] == string(UTF8BOM) {
+		_, _ = sb.WriteString(data[:len(UTF8BOM)])
+		pos = len(UTF8BOM)
+	}
+	dataBytes := []byte(data)
+	for pos < len(data) {
+		nextIdxs := defaultWordRegexp.FindStringIndex(data[pos:])
+		if nextIdxs == nil {
+			until = len(data)
+			next = until
+		} else {
+			until, next = nextIdxs[0]+pos, nextIdxs[1]+pos
+		}
+
+		// from pos until we know that the runes are not \r\t\n or even ' '
+		runes := make([]rune, 0, next-until)
+		positions := make([]int, 0, next-until+1)
+
+		for pos < until {
+			r, sz := utf8.DecodeRune(dataBytes[pos:])
+			positions = positions[:0]
+			positions = append(positions, pos, pos+sz)
+			types, confusables, _ := e.runeTypes(r)
+			if err := e.handleRunes(dataBytes, []rune{r}, positions, types, confusables, sb); err != nil {
+				return err
+			}
+			pos += sz
+		}
+
+		for i := pos; i < next; {
+			r, sz := utf8.DecodeRune(dataBytes[i:])
+			runes = append(runes, r)
+			positions = append(positions, i)
+			i += sz
+		}
+		positions = append(positions, next)
+		types, confusables, runeCounts := e.runeTypes(runes...)
+		if runeCounts.needsEscape() {
+			if err := e.handleRunes(dataBytes, runes, positions, types, confusables, sb); err != nil {
+				return err
+			}
+		} else {
+			_, _ = sb.Write(dataBytes[pos:next])
+		}
+		pos = next
+	}
+	if sb.Len() > 0 {
+		if err := e.PassthroughHTMLStreamer.Text(sb.String()); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (e *escapeStreamer) handleRunes(data []byte, runes []rune, positions []int, types []runeType, confusables []rune, sb *strings.Builder) error {
+	for i, r := range runes {
+		switch types[i] {
+		case brokenRuneType:
+			if sb.Len() > 0 {
+				if err := e.PassthroughHTMLStreamer.Text(sb.String()); err != nil {
+					return err
+				}
+				sb.Reset()
+			}
+			end := positions[i+1]
+			start := positions[i]
+			if err := e.brokenRune(data[start:end]); err != nil {
+				return err
+			}
+		case ambiguousRuneType:
+			if sb.Len() > 0 {
+				if err := e.PassthroughHTMLStreamer.Text(sb.String()); err != nil {
+					return err
+				}
+				sb.Reset()
+			}
+			if err := e.ambiguousRune(r, confusables[0]); err != nil {
+				return err
+			}
+			confusables = confusables[1:]
+		case invisibleRuneType:
+			if sb.Len() > 0 {
+				if err := e.PassthroughHTMLStreamer.Text(sb.String()); err != nil {
+					return err
+				}
+				sb.Reset()
+			}
+			if err := e.invisibleRune(r); err != nil {
+				return err
+			}
+		default:
+			_, _ = sb.WriteRune(r)
+		}
+	}
+	return nil
+}
+
+func (e *escapeStreamer) brokenRune(bs []byte) error {
+	e.escaped.Escaped = true
+	e.escaped.HasBadRunes = true
+
+	if err := e.PassthroughHTMLStreamer.StartTag("span", html.Attribute{
+		Key: "class",
+		Val: "broken-code-point",
+	}); err != nil {
+		return err
+	}
+	if err := e.PassthroughHTMLStreamer.Text(fmt.Sprintf("<%X>", bs)); err != nil {
+		return err
+	}
+
+	return e.PassthroughHTMLStreamer.EndTag("span")
+}
+
+func (e *escapeStreamer) ambiguousRune(r, c rune) error {
+	e.escaped.Escaped = true
+	e.escaped.HasAmbiguous = true
+
+	if err := e.PassthroughHTMLStreamer.StartTag("span", html.Attribute{
+		Key: "class",
+		Val: "ambiguous-code-point",
+	}, html.Attribute{
+		Key: "data-tooltip-content",
+		Val: e.locale.TrString("repo.ambiguous_character", r, c),
+	}); err != nil {
+		return err
+	}
+	if err := e.PassthroughHTMLStreamer.StartTag("span", html.Attribute{
+		Key: "class",
+		Val: "char",
+	}); err != nil {
+		return err
+	}
+	if err := e.PassthroughHTMLStreamer.Text(string(r)); err != nil {
+		return err
+	}
+	if err := e.PassthroughHTMLStreamer.EndTag("span"); err != nil {
+		return err
+	}
+
+	return e.PassthroughHTMLStreamer.EndTag("span")
+}
+
+func (e *escapeStreamer) invisibleRune(r rune) error {
+	e.escaped.Escaped = true
+	e.escaped.HasInvisible = true
+
+	if err := e.PassthroughHTMLStreamer.StartTag("span", html.Attribute{
+		Key: "class",
+		Val: "escaped-code-point",
+	}, html.Attribute{
+		Key: "data-escaped",
+		Val: fmt.Sprintf("[U+%04X]", r),
+	}); err != nil {
+		return err
+	}
+	if err := e.PassthroughHTMLStreamer.StartTag("span", html.Attribute{
+		Key: "class",
+		Val: "char",
+	}); err != nil {
+		return err
+	}
+	if err := e.PassthroughHTMLStreamer.Text(string(r)); err != nil {
+		return err
+	}
+	if err := e.PassthroughHTMLStreamer.EndTag("span"); err != nil {
+		return err
+	}
+
+	return e.PassthroughHTMLStreamer.EndTag("span")
+}
+
+type runeCountType struct {
+	numBasicRunes                int
+	numNonConfusingNonBasicRunes int
+	numAmbiguousRunes            int
+	numInvisibleRunes            int
+	numBrokenRunes               int
+}
+
+func (counts runeCountType) needsEscape() bool {
+	if counts.numBrokenRunes > 0 {
+		return true
+	}
+	if counts.numBasicRunes == 0 &&
+		counts.numNonConfusingNonBasicRunes > 0 {
+		return false
+	}
+	return counts.numAmbiguousRunes > 0 || counts.numInvisibleRunes > 0
+}
+
+type runeType int
+
+const (
+	basicASCIIRuneType runeType = iota // <- This is technically deadcode but its self-documenting so it should stay
+	brokenRuneType
+	nonBasicASCIIRuneType
+	ambiguousRuneType
+	invisibleRuneType
+)
+
+func (e *escapeStreamer) runeTypes(runes ...rune) (types []runeType, confusables []rune, runeCounts runeCountType) {
+	types = make([]runeType, len(runes))
+	for i, r := range runes {
+		var confusable rune
+		switch {
+		case r == utf8.RuneError:
+			types[i] = brokenRuneType
+			runeCounts.numBrokenRunes++
+		case r == ' ' || r == '\t' || r == '\n':
+			runeCounts.numBasicRunes++
+		case e.allowed[r]:
+			if r > 0x7e || r < 0x20 {
+				types[i] = nonBasicASCIIRuneType
+				runeCounts.numNonConfusingNonBasicRunes++
+			} else {
+				runeCounts.numBasicRunes++
+			}
+		case unicode.Is(InvisibleRanges, r):
+			types[i] = invisibleRuneType
+			runeCounts.numInvisibleRunes++
+		case unicode.IsControl(r):
+			types[i] = invisibleRuneType
+			runeCounts.numInvisibleRunes++
+		case isAmbiguous(r, &confusable, e.ambiguousTables...):
+			confusables = append(confusables, confusable)
+			types[i] = ambiguousRuneType
+			runeCounts.numAmbiguousRunes++
+		case r > 0x7e || r < 0x20:
+			types[i] = nonBasicASCIIRuneType
+			runeCounts.numNonConfusingNonBasicRunes++
+		default:
+			runeCounts.numBasicRunes++
+		}
+	}
+	return types, confusables, runeCounts
+}

modules/charset/escape_test.go

@@ -0,0 +1,181 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package charset
+
+import (
+	"regexp"
+	"strings"
+	"testing"
+
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/test"
+	"code.gitea.io/gitea/modules/translation"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type escapeControlTest struct {
+	name   string
+	text   string
+	status EscapeStatus
+	result string
+}
+
+var escapeControlTests = []escapeControlTest{
+	{
+		name: "<empty>",
+	},
+	{
+		name:   "single line western",
+		text:   "single line western",
+		result: "single line western",
+		status: EscapeStatus{},
+	},
+	{
+		name:   "multi line western",
+		text:   "single line western\nmulti line western\n",
+		result: "single line western\nmulti line western\n",
+		status: EscapeStatus{},
+	},
+	{
+		name:   "multi line western non-breaking space",
+		text:   "single line western\nmulti line western\n",
+		result: `single line<span class="escaped-code-point" data-escaped="[U+00A0]"><span class="char"> </span></span>western` + "\n" + `multi line<span class="escaped-code-point" data-escaped="[U+00A0]"><span class="char"> </span></span>western` + "\n",
+		status: EscapeStatus{Escaped: true, HasInvisible: true},
+	},
+	{
+		name:   "mixed scripts: western + japanese",
+		text:   "日属秘ぞしちゅ。Then some western.",
+		result: "日属秘ぞしちゅ。Then some western.",
+		status: EscapeStatus{},
+	},
+	{
+		name:   "japanese",
+		text:   "日属秘ぞしちゅ。",
+		result: "日属秘ぞしちゅ。",
+		status: EscapeStatus{},
+	},
+	{
+		name:   "hebrew",
+		text:   "עד תקופת יוון העתיקה היה העיסוק במתמטיקה תכליתי בלבד: היא שימשה כאוסף של נוסחאות לחישוב קרקע, אוכלוסין וכו'. פריצת הדרך של היוונים, פרט לתרומותיהם הגדולות לידע המתמטי, הייתה בלימוד המתמטיקה כשלעצמה, מתוקף ערכה הרוחני. יחסם של חלק מהיוונים הקדמונים למתמטיקה היה דתי - למשל, הכת שאסף סביבו פיתגורס האמינה כי המתמטיקה היא הבסיס לכל הדברים. היוונים נחשבים ליוצרי מושג ההוכחה המתמטית, וכן לראשונים שעסקו במתמטיקה לשם עצמה, כלומר כתחום מחקרי עיוני ומופשט ולא רק כעזר שימושי. עם זאת, לצדה",
+		result: `עד תקופת <span class="ambiguous-code-point" data-tooltip-content="repo.ambiguous_character"><span class="char">י</span></span><span class="ambiguous-code-point" data-tooltip-content="repo.ambiguous_character"><span class="char">ו</span></span><span class="ambiguous-code-point" data-tooltip-content="repo.ambiguous_character"><span class="char">ו</span></span><span class="ambiguous-code-point" data-tooltip-content="repo.ambiguous_character"><span class="char">ן</span></span> העתיקה היה העיסוק במתמטיקה תכליתי בלבד: היא שימשה כאוסף של נוסחאות לחישוב קרקע, אוכלוסין וכו&#39;. פריצת הדרך של היוונים, פרט לתרומותיהם הגדולות לידע המתמטי, הייתה בלימוד המתמטיקה כשלעצמה, מתוקף ערכה הרוחני. יחסם של חלק מהיוונים הקדמונים למתמטיקה היה דתי - למשל, הכת שאסף סביבו פיתגורס האמינה כי המתמטיקה היא הבסיס לכל הדברים. היוונים נחשבים ליוצרי מושג ההוכחה המתמטית, וכן לראשונים שעסקו במתמטיקה לשם עצמה, כלומר כתחום מחקרי עיוני ומופשט ולא רק כעזר שימושי. עם זאת, לצדה`,
+		status: EscapeStatus{Escaped: true, HasAmbiguous: true},
+	},
+	{
+		name: "more hebrew",
+		text: `בתקופה מאוחרת יותר, השתמשו היוונים בשיטת סימון מתקדמת יותר, שבה הוצגו המספרים לפי 22 אותיות האלפבית היווני. לסימון המספרים בין 1 ל-9 נקבעו תשע האותיות הראשונות, בתוספת גרש ( ' ) בצד ימין של האות, למעלה; תשע האותיות הבאות ייצגו את העשרות מ-10 עד 90, והבאות את המאות. לסימון הספרות בין 1000 ל-900,000, השתמשו היוונים באותן אותיות, אך הוסיפו לאותיות את הגרש דווקא מצד שמאל של האותיות, למטה. ממיליון ומעלה, כנראה השתמשו היוונים בשני תגים במקום אחד.
+
+			המתמטיקאי הבולט הראשון ביוון העתיקה, ויש האומרים בתולדות האנושות, הוא תאלס (624 לפנה"ס - 546 לפנה"ס בקירוב).[1] לא יהיה זה משולל יסוד להניח שהוא האדם הראשון שהוכיח משפט מתמטי, ולא רק גילה אותו. תאלס הוכיח שישרים מקבילים חותכים מצד אחד של שוקי זווית קטעים בעלי יחסים שווים (משפט תאלס הראשון), שהזווית המונחת על קוטר במעגל היא זווית ישרה (משפט תאלס השני), שהקוטר מחלק את המעגל לשני חלקים שווים, ושזוויות הבסיס במשולש שווה-שוקיים שוות זו לזו. מיוחסות לו גם שיטות למדידת גובהן של הפירמידות בעזרת מדידת צילן ולקביעת מיקומה של ספינה הנראית מן החוף.
+
+			בשנים 582 לפנה"ס עד 496 לפנה"ס, בקירוב, חי מתמטיקאי חשוב במיוחד - פיתגורס. המקורות הראשוניים עליו מועטים, וההיסטוריונים מתקשים להפריד את העובדות משכבת המסתורין והאגדות שנקשרו בו. ידוע שסביבו התקבצה האסכולה הפיתגוראית מעין כת פסבדו-מתמטית שהאמינה ש"הכל מספר", או ליתר דיוק הכל ניתן לכימות, וייחסה למספרים משמעויות מיסטיות. ככל הנראה הפיתגוראים ידעו לבנות את הגופים האפלטוניים, הכירו את הממוצע האריתמטי, הממוצע הגאומטרי והממוצע ההרמוני והגיעו להישגים חשובים נוספים. ניתן לומר שהפיתגוראים גילו את היותו של השורש הריבועי של 2, שהוא גם האלכסון בריבוע שאורך צלעותיו 1, אי רציונלי, אך תגליתם הייתה למעשה רק שהקטעים "חסרי מידה משותפת", ומושג המספר האי רציונלי מאוחר יותר.[2] אזכור ראשון לקיומם של קטעים חסרי מידה משותפת מופיע בדיאלוג "תאיטיטוס" של אפלטון, אך רעיון זה היה מוכר עוד קודם לכן, במאה החמישית לפנה"ס להיפאסוס, בן האסכולה הפיתגוראית, ואולי לפיתגורס עצמו.[3]`,
+		result: `בתקופה מאוחרת יותר, השתמשו היוונים בשיטת סימון מתקדמת יותר, שבה הוצגו המספרים לפי 22 אותיות האלפבית היווני. לסימון המספרים בין 1 ל-9 נקבעו תשע האותיות הראשונות, בתוספת גרש ( &#39; ) בצד ימין של האות, למעלה; תשע האותיות הבאות ייצגו את העשרות מ-10 עד 90, והבאות את המאות. לסימון הספרות בין 1000 ל-900,000, השתמשו היוונים באותן אותיות, אך הוסיפו לאותיות את הגרש דווקא מצד שמאל של האותיות, למטה. ממיליון ומעלה, כנראה השתמשו היוונים בשני תגים במקום אחד.
+
+			המתמטיקאי הבולט הראשון ביוון העתיקה, ויש האומרים בתולדות האנושות, הוא תאלס (624 לפנה&#34;<span class="ambiguous-code-point" data-tooltip-content="repo.ambiguous_character"><span class="char">ס</span></span> - 546 לפנה&#34;<span class="ambiguous-code-point" data-tooltip-content="repo.ambiguous_character"><span class="char">ס</span></span> בקירוב).[1] לא יהיה זה משולל יסוד להניח שהוא האדם הראשון שהוכיח משפט מתמטי, ולא רק גילה אותו. תאלס הוכיח שישרים מקבילים חותכים מצד אחד של שוקי זווית קטעים בעלי יחסים שווים (משפט תאלס הראשון), שהזווית המונחת על קוטר במעגל היא זווית ישרה (משפט תאלס השני), שהקוטר מחלק את המעגל לשני חלקים שווים, ושזוויות הבסיס במשולש שווה-שוקיים שוות זו לזו. מיוחסות לו גם שיטות למדידת גובהן של הפירמידות בעזרת מדידת צילן ולקביעת מיקומה של ספינה הנראית מן החוף.
+
+			בשנים 582 לפנה&#34;<span class="ambiguous-code-point" data-tooltip-content="repo.ambiguous_character"><span class="char">ס</span></span> עד 496 לפנה&#34;<span class="ambiguous-code-point" data-tooltip-content="repo.ambiguous_character"><span class="char">ס</span></span>, בקירוב, חי מתמטיקאי חשוב במיוחד - פיתגורס. המקורות הראשוניים עליו מועטים, וההיסטוריונים מתקשים להפריד את העובדות משכבת המסתורין והאגדות שנקשרו בו. ידוע שסביבו התקבצה האסכולה הפיתגוראית מעין כת פסבדו-מתמטית שהאמינה ש&#34;הכל מספר&#34;, או ליתר דיוק הכל ניתן לכימות, וייחסה למספרים משמעויות מיסטיות. ככל הנראה הפיתגוראים ידעו לבנות את הגופים האפלטוניים, הכירו את הממוצע האריתמטי, הממוצע הגאומטרי והממוצע ההרמוני והגיעו להישגים חשובים נוספים. ניתן לומר שהפיתגוראים גילו את היותו של השורש הריבועי של 2, שהוא גם האלכסון בריבוע שאורך צלעותיו 1, אי רציונלי, אך תגליתם הייתה למעשה רק שהקטעים &#34;חסרי מידה משותפת&#34;, ומושג המספר האי רציונלי מאוחר יותר.[2] אזכור ראשון לקיומם של קטעים חסרי מידה משותפת מופיע בדיאלוג &#34;תאיטיטוס&#34; של אפלטון, אך רעיון זה היה מוכר עוד קודם לכן, במאה החמישית לפנה&#34;<span class="ambiguous-code-point" data-tooltip-content="repo.ambiguous_character"><span class="char">ס</span></span> להיפאסוס, בן האסכולה הפיתגוראית, ואולי לפיתגורס עצמו.[3]`,
+		status: EscapeStatus{Escaped: true, HasAmbiguous: true},
+	},
+	{
+		name: "Mixed RTL+LTR",
+		text: `Many computer programs fail to display bidirectional text correctly.
+For example, the Hebrew name Sarah (שרה) is spelled: sin (ש) (which appears rightmost),
+then resh (ר), and finally heh (ה) (which should appear leftmost).`,
+		result: `Many computer programs fail to display bidirectional text correctly.
+For example, the Hebrew name Sarah (שרה) is spelled: sin (ש) (which appears rightmost),
+then resh (ר), and finally heh (ה) (which should appear leftmost).`,
+		status: EscapeStatus{},
+	},
+	{
+		name: "Mixed RTL+LTR+BIDI",
+		text: `Many computer programs fail to display bidirectional text correctly.
+			For example, the Hebrew name Sarah ` + "\u2067" + `שרה` + "\u2066\n" +
+			`sin (ש) (which appears rightmost), then resh (ר), and finally heh (ה) (which should appear leftmost).`,
+		result: `Many computer programs fail to display bidirectional text correctly.
+			For example, the Hebrew name Sarah ` + "\u2067" + `שרה` + "\u2066\n" +
+			`sin (ש) (which appears rightmost), then resh (ר), and finally heh (ה) (which should appear leftmost).`,
+		status: EscapeStatus{},
+	},
+	{
+		name:   "Accented characters",
+		text:   string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}),
+		result: string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}),
+		status: EscapeStatus{},
+	},
+	{
+		name:   "Program",
+		text:   "string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})",
+		result: "string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})",
+		status: EscapeStatus{},
+	},
+	{
+		name:   "CVE testcase",
+		text:   "if access_level != \"user\u202E \u2066// Check if admin\u2069 \u2066\" {",
+		result: `if access_level != &#34;user<span class="escaped-code-point" data-escaped="[U+202E]"><span class="char">` + "\u202e" + `</span></span> <span class="escaped-code-point" data-escaped="[U+2066]"><span class="char">` + "\u2066" + `</span></span>// Check if admin<span class="escaped-code-point" data-escaped="[U+2069]"><span class="char">` + "\u2069" + `</span></span> <span class="escaped-code-point" data-escaped="[U+2066]"><span class="char">` + "\u2066" + `</span></span>&#34; {`,
+		status: EscapeStatus{Escaped: true, HasInvisible: true},
+	},
+	{
+		name: "Mixed testcase with fail",
+		text: `Many computer programs fail to display bidirectional text correctly.
+			For example, the Hebrew name Sarah ` + "\u2067" + `שרה` + "\u2066\n" +
+			`sin (ש) (which appears rightmost), then resh (ר), and finally heh (ה) (which should appear leftmost).` +
+			"\nif access_level != \"user\u202E \u2066// Check if admin\u2069 \u2066\" {\n",
+		result: `Many computer programs fail to display bidirectional text correctly.
+			For example, the Hebrew name Sarah ` + "\u2067" + `שרה` + "\u2066\n" +
+			`sin (ש) (which appears rightmost), then resh (ר), and finally heh (ה) (which should appear leftmost).` +
+			"\n" + `if access_level != &#34;user<span class="escaped-code-point" data-escaped="[U+202E]"><span class="char">` + "\u202e" + `</span></span> <span class="escaped-code-point" data-escaped="[U+2066]"><span class="char">` + "\u2066" + `</span></span>// Check if admin<span class="escaped-code-point" data-escaped="[U+2069]"><span class="char">` + "\u2069" + `</span></span> <span class="escaped-code-point" data-escaped="[U+2066]"><span class="char">` + "\u2066" + `</span></span>&#34; {` + "\n",
+		status: EscapeStatus{Escaped: true, HasInvisible: true},
+	},
+	{
+		// UTF-8/16/32 all use the same codepoint for BOM
+		// Gitea could read UTF-16/32 content and convert into UTF-8 internally then render it, so we only process UTF-8 internally
+		name:   "UTF BOM",
+		text:   "\xef\xbb\xbftest",
+		result: "\xef\xbb\xbftest",
+		status: EscapeStatus{},
+	},
+}
+
+func TestEscapeControlReader(t *testing.T) {
+	// add some control characters to the tests
+	tests := make([]escapeControlTest, 0, len(escapeControlTests)*3)
+	copy(tests, escapeControlTests)
+
+	// if there is a BOM, we should keep the BOM
+	addPrefix := func(prefix, s string) string {
+		if strings.HasPrefix(s, "\xef\xbb\xbf") {
+			return s[:3] + prefix + s[3:]
+		}
+		return prefix + s
+	}
+	for _, test := range escapeControlTests {
+		test.name += " (+Control)"
+		test.text = addPrefix("\u001E", test.text)
+		test.result = addPrefix(`<span class="escaped-code-point" data-escaped="[U+001E]"><span class="char">`+"\u001e"+`</span></span>`, test.result)
+		test.status.Escaped = true
+		test.status.HasInvisible = true
+		tests = append(tests, test)
+	}
+
+	re := regexp.MustCompile(`repo.ambiguous_character:\d+,\d+`) // simplify the output for the tests, remove the translation variants
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			output := &strings.Builder{}
+			status, err := EscapeControlReader(strings.NewReader(tt.text), output, &translation.MockLocale{})
+			assert.NoError(t, err)
+			assert.Equal(t, tt.status, *status)
+			outStr := output.String()
+			outStr = re.ReplaceAllString(outStr, "repo.ambiguous_character")
+			assert.Equal(t, tt.result, outStr)
+		})
+	}
+}
+
+func TestSettingAmbiguousUnicodeDetection(t *testing.T) {
+	defer test.MockVariableValue(&setting.UI.AmbiguousUnicodeDetection, true)()
+	_, out := EscapeControlHTML("a test", &translation.MockLocale{})
+	assert.EqualValues(t, `a<span class="escaped-code-point" data-escaped="[U+00A0]"><span class="char"> </span></span>test`, out)
+	setting.UI.AmbiguousUnicodeDetection = false
+	_, out = EscapeControlHTML("a test", &translation.MockLocale{})
+	assert.EqualValues(t, `a test`, out)
+}

modules/charset/htmlstream.go

@@ -0,0 +1,200 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package charset
+
+import (
+	"fmt"
+	"io"
+
+	"golang.org/x/net/html"
+)
+
+// HTMLStreamer represents a SAX-like interface for HTML
+type HTMLStreamer interface {
+	Error(err error) error
+	Doctype(data string) error
+	Comment(data string) error
+	StartTag(data string, attrs ...html.Attribute) error
+	SelfClosingTag(data string, attrs ...html.Attribute) error
+	EndTag(data string) error
+	Text(data string) error
+}
+
+// PassthroughHTMLStreamer is a passthrough streamer
+type PassthroughHTMLStreamer struct {
+	next HTMLStreamer
+}
+
+func NewPassthroughStreamer(next HTMLStreamer) *PassthroughHTMLStreamer {
+	return &PassthroughHTMLStreamer{next: next}
+}
+
+var _ (HTMLStreamer) = &PassthroughHTMLStreamer{}
+
+// Error tells the next streamer in line that there is an error
+func (p *PassthroughHTMLStreamer) Error(err error) error {
+	return p.next.Error(err)
+}
+
+// Doctype tells the next streamer what the doctype is
+func (p *PassthroughHTMLStreamer) Doctype(data string) error {
+	return p.next.Doctype(data)
+}
+
+// Comment tells the next streamer there is a comment
+func (p *PassthroughHTMLStreamer) Comment(data string) error {
+	return p.next.Comment(data)
+}
+
+// StartTag tells the next streamer there is a starting tag
+func (p *PassthroughHTMLStreamer) StartTag(data string, attrs ...html.Attribute) error {
+	return p.next.StartTag(data, attrs...)
+}
+
+// SelfClosingTag tells the next streamer there is a self-closing tag
+func (p *PassthroughHTMLStreamer) SelfClosingTag(data string, attrs ...html.Attribute) error {
+	return p.next.SelfClosingTag(data, attrs...)
+}
+
+// EndTag tells the next streamer there is a end tag
+func (p *PassthroughHTMLStreamer) EndTag(data string) error {
+	return p.next.EndTag(data)
+}
+
+// Text tells the next streamer there is a text
+func (p *PassthroughHTMLStreamer) Text(data string) error {
+	return p.next.Text(data)
+}
+
+// HTMLStreamWriter acts as a writing sink
+type HTMLStreamerWriter struct {
+	io.Writer
+	err error
+}
+
+// Write implements io.Writer
+func (h *HTMLStreamerWriter) Write(data []byte) (int, error) {
+	if h.err != nil {
+		return 0, h.err
+	}
+	return h.Writer.Write(data)
+}
+
+// Write implements io.StringWriter
+func (h *HTMLStreamerWriter) WriteString(data string) (int, error) {
+	if h.err != nil {
+		return 0, h.err
+	}
+	return h.Writer.Write([]byte(data))
+}
+
+// Error tells the next streamer in line that there is an error
+func (h *HTMLStreamerWriter) Error(err error) error {
+	if h.err == nil {
+		h.err = err
+	}
+	return h.err
+}
+
+// Doctype tells the next streamer what the doctype is
+func (h *HTMLStreamerWriter) Doctype(data string) error {
+	_, h.err = h.WriteString("<!DOCTYPE " + data + ">")
+	return h.err
+}
+
+// Comment tells the next streamer there is a comment
+func (h *HTMLStreamerWriter) Comment(data string) error {
+	_, h.err = h.WriteString("<!--" + data + "-->")
+	return h.err
+}
+
+// StartTag tells the next streamer there is a starting tag
+func (h *HTMLStreamerWriter) StartTag(data string, attrs ...html.Attribute) error {
+	return h.startTag(data, attrs, false)
+}
+
+// SelfClosingTag tells the next streamer there is a self-closing tag
+func (h *HTMLStreamerWriter) SelfClosingTag(data string, attrs ...html.Attribute) error {
+	return h.startTag(data, attrs, true)
+}
+
+func (h *HTMLStreamerWriter) startTag(data string, attrs []html.Attribute, selfclosing bool) error {
+	if _, h.err = h.WriteString("<" + data); h.err != nil {
+		return h.err
+	}
+	for _, attr := range attrs {
+		if _, h.err = h.WriteString(" " + attr.Key + "=\"" + html.EscapeString(attr.Val) + "\""); h.err != nil {
+			return h.err
+		}
+	}
+	if selfclosing {
+		if _, h.err = h.WriteString("/>"); h.err != nil {
+			return h.err
+		}
+	} else {
+		if _, h.err = h.WriteString(">"); h.err != nil {
+			return h.err
+		}
+	}
+	return h.err
+}
+
+// EndTag tells the next streamer there is a end tag
+func (h *HTMLStreamerWriter) EndTag(data string) error {
+	_, h.err = h.WriteString("</" + data + ">")
+	return h.err
+}
+
+// Text tells the next streamer there is a text
+func (h *HTMLStreamerWriter) Text(data string) error {
+	_, h.err = h.WriteString(html.EscapeString(data))
+	return h.err
+}
+
+// StreamHTML streams an html to a provided streamer
+func StreamHTML(source io.Reader, streamer HTMLStreamer) error {
+	tokenizer := html.NewTokenizer(source)
+	for {
+		tt := tokenizer.Next()
+		switch tt {
+		case html.ErrorToken:
+			if tokenizer.Err() != io.EOF {
+				return tokenizer.Err()
+			}
+			return nil
+		case html.DoctypeToken:
+			token := tokenizer.Token()
+			if err := streamer.Doctype(token.Data); err != nil {
+				return err
+			}
+		case html.CommentToken:
+			token := tokenizer.Token()
+			if err := streamer.Comment(token.Data); err != nil {
+				return err
+			}
+		case html.StartTagToken:
+			token := tokenizer.Token()
+			if err := streamer.StartTag(token.Data, token.Attr...); err != nil {
+				return err
+			}
+		case html.SelfClosingTagToken:
+			token := tokenizer.Token()
+			if err := streamer.StartTag(token.Data, token.Attr...); err != nil {
+				return err
+			}
+		case html.EndTagToken:
+			token := tokenizer.Token()
+			if err := streamer.EndTag(token.Data); err != nil {
+				return err
+			}
+		case html.TextToken:
+			token := tokenizer.Token()
+			if err := streamer.Text(token.Data); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("unknown type of token: %d", tt)
+		}
+	}
+}

modules/charset/invisible/generate.go

@@ -0,0 +1,121 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/format"
+	"os"
+	"text/template"
+
+	"golang.org/x/text/unicode/rangetable"
+)
+
+// InvisibleRunes these are runes that vscode has assigned to be invisible
+// See https://github.com/hediet/vscode-unicode-data
+var InvisibleRunes = []rune{
+	9, 10, 11, 12, 13, 32, 127, 160, 173, 847, 1564, 4447, 4448, 6068, 6069, 6155, 6156, 6157, 6158, 7355, 7356, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200, 8201, 8202, 8203, 8204, 8205, 8206, 8207, 8234, 8235, 8236, 8237, 8238, 8239, 8287, 8288, 8289, 8290, 8291, 8292, 8293, 8294, 8295, 8296, 8297, 8298, 8299, 8300, 8301, 8302, 8303, 10240, 12288, 12644, 65024, 65025, 65026, 65027, 65028, 65029, 65030, 65031, 65032, 65033, 65034, 65035, 65036, 65037, 65038, 65039, 65279, 65440, 65520, 65521, 65522, 65523, 65524, 65525, 65526, 65527, 65528, 65532, 78844, 119155, 119156, 119157, 119158, 119159, 119160, 119161, 119162, 917504, 917505, 917506, 917507, 917508, 917509, 917510, 917511, 917512, 917513, 917514, 917515, 917516, 917517, 917518, 917519, 917520, 917521, 917522, 917523, 917524, 917525, 917526, 917527, 917528, 917529, 917530, 917531, 917532, 917533, 917534, 917535, 917536, 917537, 917538, 917539, 917540, 917541, 917542, 917543, 917544, 917545, 917546, 917547, 917548, 917549, 917550, 917551, 917552, 917553, 917554, 917555, 917556, 917557, 917558, 917559, 917560, 917561, 917562, 917563, 917564, 917565, 917566, 917567, 917568, 917569, 917570, 917571, 917572, 917573, 917574, 917575, 917576, 917577, 917578, 917579, 917580, 917581, 917582, 917583, 917584, 917585, 917586, 917587, 917588, 917589, 917590, 917591, 917592, 917593, 917594, 917595, 917596, 917597, 917598, 917599, 917600, 917601, 917602, 917603, 917604, 917605, 917606, 917607, 917608, 917609, 917610, 917611, 917612, 917613, 917614, 917615, 917616, 917617, 917618, 917619, 917620, 917621, 917622, 917623, 917624, 917625, 917626, 917627, 917628, 917629, 917630, 917631, 917760, 917761, 917762, 917763, 917764, 917765, 917766, 917767, 917768, 917769, 917770, 917771, 917772, 917773, 917774, 917775, 917776, 917777, 917778, 917779, 917780, 917781, 917782, 917783, 917784, 917785, 917786, 917787, 917788, 917789, 917790, 917791, 917792, 917793, 917794, 917795, 917796, 917797, 917798, 917799, 917800, 917801, 917802, 917803, 917804, 917805, 917806, 917807, 917808, 917809, 917810, 917811, 917812, 917813, 917814, 917815, 917816, 917817, 917818, 917819, 917820, 917821, 917822, 917823, 917824, 917825, 917826, 917827, 917828, 917829, 917830, 917831, 917832, 917833, 917834, 917835, 917836, 917837, 917838, 917839, 917840, 917841, 917842, 917843, 917844, 917845, 917846, 917847, 917848, 917849, 917850, 917851, 917852, 917853, 917854, 917855, 917856, 917857, 917858, 917859, 917860, 917861, 917862, 917863, 917864, 917865, 917866, 917867, 917868, 917869, 917870, 917871, 917872, 917873, 917874, 917875, 917876, 917877, 917878, 917879, 917880, 917881, 917882, 917883, 917884, 917885, 917886, 917887, 917888, 917889, 917890, 917891, 917892, 917893, 917894, 917895, 917896, 917897, 917898, 917899, 917900, 917901, 917902, 917903, 917904, 917905, 917906, 917907, 917908, 917909, 917910, 917911, 917912, 917913, 917914, 917915, 917916, 917917, 917918, 917919, 917920, 917921, 917922, 917923, 917924, 917925, 917926, 917927, 917928, 917929, 917930, 917931, 917932, 917933, 917934, 917935, 917936, 917937, 917938, 917939, 917940, 917941, 917942, 917943, 917944, 917945, 917946, 917947, 917948, 917949, 917950, 917951, 917952, 917953, 917954, 917955, 917956, 917957, 917958, 917959, 917960, 917961, 917962, 917963, 917964, 917965, 917966, 917967, 917968, 917969, 917970, 917971, 917972, 917973, 917974, 917975, 917976, 917977, 917978, 917979, 917980, 917981, 917982, 917983, 917984, 917985, 917986, 917987, 917988, 917989, 917990, 917991, 917992, 917993, 917994, 917995, 917996, 917997, 917998, 917999,
+}
+
+var verbose bool
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, `%s: Generate InvisibleRunesRange
+
+Usage: %[1]s [-v] [-o output.go]
+`, os.Args[0])
+		flag.PrintDefaults()
+	}
+
+	output := ""
+	flag.BoolVar(&verbose, "v", false, "verbose output")
+	flag.StringVar(&output, "o", "invisible_gen.go", "file to output to")
+	flag.Parse()
+
+	// First we filter the runes to remove
+	// <space><tab><newline>
+	filtered := make([]rune, 0, len(InvisibleRunes))
+	for _, r := range InvisibleRunes {
+		if r == ' ' || r == '\t' || r == '\n' {
+			continue
+		}
+		filtered = append(filtered, r)
+	}
+
+	table := rangetable.New(filtered...)
+	if err := runTemplate(generatorTemplate, output, table); err != nil {
+		fatalf("Unable to run template: %v", err)
+	}
+}
+
+func runTemplate(t *template.Template, filename string, data any) error {
+	buf := bytes.NewBuffer(nil)
+	if err := t.Execute(buf, data); err != nil {
+		return fmt.Errorf("unable to execute template: %w", err)
+	}
+	bs, err := format.Source(buf.Bytes())
+	if err != nil {
+		verbosef("Bad source:\n%s", buf.String())
+		return fmt.Errorf("unable to format source: %w", err)
+	}
+
+	old, err := os.ReadFile(filename)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to read old file %s because %w", filename, err)
+	} else if err == nil {
+		if bytes.Equal(bs, old) {
+			// files are the same don't rewrite it.
+			return nil
+		}
+	}
+
+	file, err := os.Create(filename)
+	if err != nil {
+		return fmt.Errorf("failed to create file %s because %w", filename, err)
+	}
+	defer file.Close()
+	_, err = file.Write(bs)
+	if err != nil {
+		return fmt.Errorf("unable to write generated source: %w", err)
+	}
+	return nil
+}
+
+var generatorTemplate = template.Must(template.New("invisibleTemplate").Parse(`// This file is generated by modules/charset/invisible/generate.go DO NOT EDIT
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+
+package charset
+
+import "unicode"
+
+var InvisibleRanges = &unicode.RangeTable{
+	R16: []unicode.Range16{
+{{range .R16 }}		{Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}},
+{{end}}	},
+	R32: []unicode.Range32{
+{{range .R32}}		{Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}},
+{{end}}	},
+	LatinOffset: {{.LatinOffset}},
+}
+`))
+
+func logf(format string, args ...any) {
+	fmt.Fprintf(os.Stderr, format+"\n", args...)
+}
+
+func verbosef(format string, args ...any) {
+	if verbose {
+		logf(format, args...)
+	}
+}
+
+func fatalf(format string, args ...any) {
+	logf("fatal: "+format+"\n", args...)
+	os.Exit(1)
+}

modules/charset/invisible_gen.go

@@ -0,0 +1,36 @@
+// This file is generated by modules/charset/invisible/generate.go DO NOT EDIT
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package charset
+
+import "unicode"
+
+var InvisibleRanges = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{Lo: 11, Hi: 13, Stride: 1},
+		{Lo: 127, Hi: 160, Stride: 33},
+		{Lo: 173, Hi: 847, Stride: 674},
+		{Lo: 1564, Hi: 4447, Stride: 2883},
+		{Lo: 4448, Hi: 6068, Stride: 1620},
+		{Lo: 6069, Hi: 6155, Stride: 86},
+		{Lo: 6156, Hi: 6158, Stride: 1},
+		{Lo: 7355, Hi: 7356, Stride: 1},
+		{Lo: 8192, Hi: 8207, Stride: 1},
+		{Lo: 8234, Hi: 8239, Stride: 1},
+		{Lo: 8287, Hi: 8303, Stride: 1},
+		{Lo: 10240, Hi: 12288, Stride: 2048},
+		{Lo: 12644, Hi: 65024, Stride: 52380},
+		{Lo: 65025, Hi: 65039, Stride: 1},
+		{Lo: 65279, Hi: 65440, Stride: 161},
+		{Lo: 65520, Hi: 65528, Stride: 1},
+		{Lo: 65532, Hi: 65532, Stride: 1},
+	},
+	R32: []unicode.Range32{
+		{Lo: 78844, Hi: 119155, Stride: 40311},
+		{Lo: 119156, Hi: 119162, Stride: 1},
+		{Lo: 917504, Hi: 917631, Stride: 1},
+		{Lo: 917760, Hi: 917999, Stride: 1},
+	},
+	LatinOffset: 2,
+}

modules/commitstatus/commit_status.go

@@ -0,0 +1,81 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package commitstatus
+
+// CommitStatusState holds the state of a CommitStatus
+// swagger:enum CommitStatusState
+type CommitStatusState string //nolint:revive // export stutter
+
+const (
+	// CommitStatusPending is for when the CommitStatus is Pending
+	CommitStatusPending CommitStatusState = "pending"
+	// CommitStatusSuccess is for when the CommitStatus is Success
+	CommitStatusSuccess CommitStatusState = "success"
+	// CommitStatusError is for when the CommitStatus is Error
+	CommitStatusError CommitStatusState = "error"
+	// CommitStatusFailure is for when the CommitStatus is Failure
+	CommitStatusFailure CommitStatusState = "failure"
+	// CommitStatusWarning is for when the CommitStatus is Warning
+	CommitStatusWarning CommitStatusState = "warning"
+	// CommitStatusSkipped is for when CommitStatus is Skipped
+	CommitStatusSkipped CommitStatusState = "skipped"
+)
+
+func (css CommitStatusState) String() string {
+	return string(css)
+}
+
+// IsPending represents if commit status state is pending
+func (css CommitStatusState) IsPending() bool {
+	return css == CommitStatusPending
+}
+
+// IsSuccess represents if commit status state is success
+func (css CommitStatusState) IsSuccess() bool {
+	return css == CommitStatusSuccess
+}
+
+// IsError represents if commit status state is error
+func (css CommitStatusState) IsError() bool {
+	return css == CommitStatusError
+}
+
+// IsFailure represents if commit status state is failure
+func (css CommitStatusState) IsFailure() bool {
+	return css == CommitStatusFailure
+}
+
+// IsWarning represents if commit status state is warning
+func (css CommitStatusState) IsWarning() bool {
+	return css == CommitStatusWarning
+}
+
+// IsSkipped represents if commit status state is skipped
+func (css CommitStatusState) IsSkipped() bool {
+	return css == CommitStatusSkipped
+}
+
+type CommitStatusStates []CommitStatusState //nolint:revive // export stutter
+
+// According to https://docs.github.com/en/rest/commits/statuses?apiVersion=2022-11-28#get-the-combined-status-for-a-specific-reference
+// > Additionally, a combined state is returned. The state is one of:
+// > failure if any of the contexts report as error or failure
+// > pending if there are no statuses or a context is pending
+// > success if the latest status for all contexts is success
+func (css CommitStatusStates) Combine() CommitStatusState {
+	successCnt := 0
+	for _, state := range css {
+		switch {
+		case state.IsError() || state.IsFailure():
+			return CommitStatusFailure
+		case state.IsPending():
+		case state.IsSuccess() || state.IsWarning() || state.IsSkipped():
+			successCnt++
+		}
+	}
+	if successCnt > 0 && successCnt == len(css) {
+		return CommitStatusSuccess
+	}
+	return CommitStatusPending
+}

modules/commitstatus/commit_status_test.go

@@ -0,0 +1,201 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package commitstatus
+
+import "testing"
+
+func TestCombine(t *testing.T) {
+	tests := []struct {
+		name     string
+		states   CommitStatusStates
+		expected CommitStatusState
+	}{
+		// 0 states
+		{
+			name:     "empty",
+			states:   CommitStatusStates{},
+			expected: CommitStatusPending,
+		},
+		// 1 state
+		{
+			name:     "pending",
+			states:   CommitStatusStates{CommitStatusPending},
+			expected: CommitStatusPending,
+		},
+		{
+			name:     "success",
+			states:   CommitStatusStates{CommitStatusSuccess},
+			expected: CommitStatusSuccess,
+		},
+		{
+			name:     "error",
+			states:   CommitStatusStates{CommitStatusError},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "failure",
+			states:   CommitStatusStates{CommitStatusFailure},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "warning",
+			states:   CommitStatusStates{CommitStatusWarning},
+			expected: CommitStatusSuccess,
+		},
+		// 2 states
+		{
+			name:     "pending and success",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusSuccess},
+			expected: CommitStatusPending,
+		},
+		{
+			name:     "pending and error",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusError},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "pending and failure",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusFailure},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "pending and warning",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusWarning},
+			expected: CommitStatusPending,
+		},
+		{
+			name:     "success and error",
+			states:   CommitStatusStates{CommitStatusSuccess, CommitStatusError},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "success and failure",
+			states:   CommitStatusStates{CommitStatusSuccess, CommitStatusFailure},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "success and warning",
+			states:   CommitStatusStates{CommitStatusSuccess, CommitStatusWarning},
+			expected: CommitStatusSuccess,
+		},
+		{
+			name:     "error and failure",
+			states:   CommitStatusStates{CommitStatusError, CommitStatusFailure},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "error and warning",
+			states:   CommitStatusStates{CommitStatusError, CommitStatusWarning},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "failure and warning",
+			states:   CommitStatusStates{CommitStatusFailure, CommitStatusWarning},
+			expected: CommitStatusFailure,
+		},
+		// 3 states
+		{
+			name:     "pending, success and warning",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusWarning},
+			expected: CommitStatusPending,
+		},
+		{
+			name:     "pending, success and error",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusError},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "pending, success and failure",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusFailure},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "pending, error and failure",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusError, CommitStatusFailure},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "success, error and warning",
+			states:   CommitStatusStates{CommitStatusSuccess, CommitStatusError, CommitStatusWarning},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "success, failure and warning",
+			states:   CommitStatusStates{CommitStatusSuccess, CommitStatusFailure, CommitStatusWarning},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "error, failure and warning",
+			states:   CommitStatusStates{CommitStatusError, CommitStatusFailure, CommitStatusWarning},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "success, warning and skipped",
+			states:   CommitStatusStates{CommitStatusSuccess, CommitStatusWarning, CommitStatusSkipped},
+			expected: CommitStatusSuccess,
+		},
+		// All success
+		{
+			name:     "all success",
+			states:   CommitStatusStates{CommitStatusSuccess, CommitStatusSuccess, CommitStatusSuccess},
+			expected: CommitStatusSuccess,
+		},
+		// All pending
+		{
+			name:     "all pending",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusPending, CommitStatusPending},
+			expected: CommitStatusPending,
+		},
+		{
+			name:     "all skipped",
+			states:   CommitStatusStates{CommitStatusSkipped, CommitStatusSkipped, CommitStatusSkipped},
+			expected: CommitStatusSuccess,
+		},
+		// 4 states
+		{
+			name:     "pending, success, error and warning",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusError, CommitStatusWarning},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "pending, success, failure and warning",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusFailure, CommitStatusWarning},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "pending, error, failure and warning",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusError, CommitStatusFailure, CommitStatusWarning},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "success, error, failure and warning",
+			states:   CommitStatusStates{CommitStatusSuccess, CommitStatusError, CommitStatusFailure, CommitStatusWarning},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "mixed states",
+			states:   CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusError, CommitStatusWarning},
+			expected: CommitStatusFailure,
+		},
+		{
+			name:     "mixed states with all success",
+			states:   CommitStatusStates{CommitStatusSuccess, CommitStatusSuccess, CommitStatusPending, CommitStatusWarning},
+			expected: CommitStatusPending,
+		},
+		{
+			name:     "all success with warning",
+			states:   CommitStatusStates{CommitStatusSuccess, CommitStatusSuccess, CommitStatusSuccess, CommitStatusWarning},
+			expected: CommitStatusSuccess,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := tt.states.Combine()
+			if result != tt.expected {
+				t.Errorf("expected %v, got %v", tt.expected, result)
+			}
+		})
+	}
+}

modules/container/filter.go

@@ -0,0 +1,21 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package container
+
+import "slices"
+
+// FilterSlice ranges over the slice and calls include() for each element.
+// If the second returned value is true, the first returned value will be included in the resulting
+// slice (after deduplication).
+func FilterSlice[E any, T comparable](s []E, include func(E) (T, bool)) []T {
+	filtered := make([]T, 0, len(s)) // slice will be clipped before returning
+	seen := make(map[T]bool, len(s))
+	for i := range s {
+		if v, ok := include(s[i]); ok && !seen[v] {
+			filtered = append(filtered, v)
+			seen[v] = true
+		}
+	}
+	return slices.Clip(filtered)
+}

modules/container/filter_test.go

@@ -0,0 +1,28 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package container
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterMapUnique(t *testing.T) {
+	result := FilterSlice([]int{
+		0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
+	}, func(i int) (int, bool) {
+		switch i {
+		case 0:
+			return 0, true // included later
+		case 1:
+			return 0, true // duplicate of previous (should be ignored)
+		case 2:
+			return 2, false // not included
+		default:
+			return i, true
+		}
+	})
+	assert.Equal(t, []int{0, 3, 4, 5, 6, 7, 8, 9}, result)
+}

modules/container/set.go

@@ -0,0 +1,71 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package container
+
+import "maps"
+
+type Set[T comparable] map[T]struct{}
+
+// SetOf creates a set and adds the specified elements to it.
+func SetOf[T comparable](values ...T) Set[T] {
+	s := make(Set[T], len(values))
+	s.AddMultiple(values...)
+	return s
+}
+
+// Add adds the specified element to a set.
+// Returns true if the element is added; false if the element is already present.
+func (s Set[T]) Add(value T) bool {
+	if _, has := s[value]; !has {
+		s[value] = struct{}{}
+		return true
+	}
+	return false
+}
+
+// AddMultiple adds the specified elements to a set.
+func (s Set[T]) AddMultiple(values ...T) {
+	for _, value := range values {
+		s.Add(value)
+	}
+}
+
+// Contains determines whether a set contains all these elements.
+// Returns true if the set contains all these elements; otherwise, false.
+func (s Set[T]) Contains(values ...T) bool {
+	ret := true
+	for _, value := range values {
+		_, has := s[value]
+		ret = ret && has
+	}
+	return ret
+}
+
+// Remove removes the specified element.
+// Returns true if the element is successfully found and removed; otherwise, false.
+func (s Set[T]) Remove(value T) bool {
+	if _, has := s[value]; has {
+		delete(s, value)
+		return true
+	}
+	return false
+}
+
+// Values gets a list of all elements in the set.
+func (s Set[T]) Values() []T {
+	keys := make([]T, 0, len(s))
+	for k := range s {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+// Union constructs a new set that is the union of the provided sets
+func (s Set[T]) Union(sets ...Set[T]) Set[T] {
+	newSet := maps.Clone(s)
+	for i := range sets {
+		maps.Copy(newSet, sets[i])
+	}
+	return newSet
+}

modules/container/set_test.go

@@ -0,0 +1,38 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package container
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSet(t *testing.T) {
+	s := make(Set[string])
+
+	assert.True(t, s.Add("key1"))
+	assert.False(t, s.Add("key1"))
+	assert.True(t, s.Add("key2"))
+
+	assert.True(t, s.Contains("key1"))
+	assert.True(t, s.Contains("key2"))
+	assert.True(t, s.Contains("key1", "key2"))
+	assert.False(t, s.Contains("key3"))
+	assert.False(t, s.Contains("key1", "key3"))
+
+	assert.True(t, s.Remove("key2"))
+	assert.False(t, s.Contains("key2"))
+
+	assert.False(t, s.Remove("key3"))
+
+	s.AddMultiple("key4", "key5")
+	assert.True(t, s.Contains("key4"))
+	assert.True(t, s.Contains("key5"))
+
+	s = SetOf("key6", "key7")
+	assert.False(t, s.Contains("key1"))
+	assert.True(t, s.Contains("key6"))
+	assert.True(t, s.Contains("key7"))
+}

modules/csv/csv.go

@@ -0,0 +1,151 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package csv
+
+import (
+	"bytes"
+	stdcsv "encoding/csv"
+	"io"
+	"path"
+	"regexp"
+	"strings"
+
+	"code.gitea.io/gitea/modules/markup"
+	"code.gitea.io/gitea/modules/translation"
+	"code.gitea.io/gitea/modules/util"
+)
+
+const (
+	maxLines        = 10
+	guessSampleSize = 1e4 // 10k
+)
+
+// CreateReader creates a csv.Reader with the given delimiter.
+func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
+	rd := stdcsv.NewReader(input)
+	rd.Comma = delimiter
+	if delimiter != '\t' && delimiter != ' ' {
+		// TrimLeadingSpace can't be true when delimiter is a tab or a space as the value for a column might be empty,
+		// thus would change `\t\t` to just `\t` or `  ` (two spaces) to just ` ` (single space)
+		rd.TrimLeadingSpace = true
+	}
+	// Don't force validation of every row to have the same number of entries as the first row.
+	rd.FieldsPerRecord = -1
+	return rd
+}
+
+// CreateReaderAndDetermineDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
+// Reads at most guessSampleSize bytes.
+func CreateReaderAndDetermineDelimiter(ctx *markup.RenderContext, rd io.Reader) (*stdcsv.Reader, error) {
+	data := make([]byte, guessSampleSize)
+	size, err := util.ReadAtMost(rd, data)
+	if err != nil {
+		return nil, err
+	}
+
+	return CreateReader(
+		io.MultiReader(bytes.NewReader(data[:size]), rd),
+		determineDelimiter(ctx, data[:size]),
+	), nil
+}
+
+// determineDelimiter takes a RenderContext and if it isn't nil and the Filename has an extension that specifies the delimiter,
+// it is used as the delimiter. Otherwise we call guessDelimiter with the data passed
+func determineDelimiter(ctx *markup.RenderContext, data []byte) rune {
+	extension := ".csv"
+	if ctx != nil {
+		extension = strings.ToLower(path.Ext(ctx.RenderOptions.RelativePath))
+	}
+
+	var delimiter rune
+	switch extension {
+	case ".tsv":
+		delimiter = '\t'
+	case ".psv":
+		delimiter = '|'
+	default:
+		delimiter = guessDelimiter(data)
+	}
+
+	return delimiter
+}
+
+// quoteRegexp follows the RFC-4180 CSV standard for when double-quotes are used to enclose fields, then a double-quote appearing inside a
+// field must be escaped by preceding it with another double quote. https://www.ietf.org/rfc/rfc4180.txt
+// This finds all quoted strings that have escaped quotes.
+var quoteRegexp = regexp.MustCompile(`"[^"]*"`)
+
+// removeQuotedStrings uses the quoteRegexp to remove all quoted strings so that we can reliably have each row on one line
+// (quoted strings often have new lines within the string)
+func removeQuotedString(text string) string {
+	return quoteRegexp.ReplaceAllLiteralString(text, "")
+}
+
+// guessDelimiter takes up to maxLines of the CSV text, iterates through the possible delimiters, and sees if the CSV Reader reads it without throwing any errors.
+// If more than one delimiter passes, the delimiter that results in the most columns is returned.
+func guessDelimiter(data []byte) rune {
+	delimiter := guessFromBeforeAfterQuotes(data)
+	if delimiter != 0 {
+		return delimiter
+	}
+
+	// Removes quoted values so we don't have columns with new lines in them
+	text := removeQuotedString(string(data))
+
+	// Make the text just be maxLines or less, ignoring truncated lines
+	lines := strings.SplitN(text, "\n", maxLines+1) // Will contain at least one line, and if there are more than MaxLines, the last item holds the rest of the lines
+	if len(lines) > maxLines {
+		// If the length of lines is > maxLines we know we have the max number of lines, trim it to maxLines
+		lines = lines[:maxLines]
+	} else if len(lines) > 1 && len(data) >= guessSampleSize {
+		// Even with data >= guessSampleSize, we don't have maxLines + 1 (no extra lines, must have really long lines)
+		// thus the last line is probably have a truncated line. Drop the last line if len(lines) > 1
+		lines = lines[:len(lines)-1]
+	}
+
+	// Put lines back together as a string
+	text = strings.Join(lines, "\n")
+
+	delimiters := []rune{',', '\t', ';', '|', '@'}
+	validDelim := delimiters[0]
+	validDelimColCount := 0
+	for _, delim := range delimiters {
+		csvReader := stdcsv.NewReader(strings.NewReader(text))
+		csvReader.Comma = delim
+		if rows, err := csvReader.ReadAll(); err == nil && len(rows) > 0 && len(rows[0]) > validDelimColCount {
+			validDelim = delim
+			validDelimColCount = len(rows[0])
+		}
+	}
+	return validDelim
+}
+
+// FormatError converts csv errors into readable messages.
+func FormatError(err error, locale translation.Locale) (string, error) {
+	if perr, ok := err.(*stdcsv.ParseError); ok {
+		if perr.Err == stdcsv.ErrFieldCount {
+			return locale.TrString("repo.error.csv.invalid_field_count", perr.Line), nil
+		}
+		return locale.TrString("repo.error.csv.unexpected", perr.Line, perr.Column), nil
+	}
+
+	return "", err
+}
+
+// Looks for possible delimiters right before or after (with spaces after the former) double quotes with closing quotes
+var beforeAfterQuotes = regexp.MustCompile(`([,@\t;|]{0,1}) *(?:"[^"]*")+([,@\t;|]{0,1})`)
+
+// guessFromBeforeAfterQuotes guesses the limiter by finding a double quote that has a valid delimiter before it and a closing quote,
+// or a double quote with a closing quote and a valid delimiter after it
+func guessFromBeforeAfterQuotes(data []byte) rune {
+	rs := beforeAfterQuotes.FindStringSubmatch(string(data)) // returns first match, or nil if none
+	if rs != nil {
+		if rs[1] != "" {
+			return rune(rs[1][0]) // delimiter found left of quoted string
+		} else if rs[2] != "" {
+			return rune(rs[2][0]) // delimiter found right of quoted string
+		}
+	}
+	return 0 // no match found
+}

modules/csv/csv_test.go

@@ -0,0 +1,588 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package csv
+
+import (
+	"bytes"
+	"encoding/csv"
+	"io"
+	"strconv"
+	"strings"
+	"testing"
+
+	"code.gitea.io/gitea/modules/markup"
+	"code.gitea.io/gitea/modules/translation"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCreateReader(t *testing.T) {
+	rd := CreateReader(bytes.NewReader([]byte{}), ',')
+	assert.Equal(t, ',', rd.Comma)
+}
+
+func decodeSlashes(t *testing.T, s string) string {
+	s = strings.ReplaceAll(s, "\n", "\\n")
+	s = strings.ReplaceAll(s, "\"", "\\\"")
+	decoded, err := strconv.Unquote(`"` + s + `"`)
+	assert.NoError(t, err, "unable to decode string")
+	return decoded
+}
+
+func TestCreateReaderAndDetermineDelimiter(t *testing.T) {
+	cases := []struct {
+		csv               string
+		expectedRows      [][]string
+		expectedDelimiter rune
+	}{
+		// case 0 - semicolon delimited
+		{
+			csv: `a;b;c
+1;2;3
+4;5;6`,
+			expectedRows: [][]string{
+				{"a", "b", "c"},
+				{"1", "2", "3"},
+				{"4", "5", "6"},
+			},
+			expectedDelimiter: ';',
+		},
+		// case 1 - tab delimited with empty fields
+		{
+			csv: `col1	col2	col3
+a,	b	c
+	e	f
+g	h	i
+j		l
+m	n,\t
+p	q	r
+		u
+v	w	x
+y\t\t
+		`,
+			expectedRows: [][]string{
+				{"col1", "col2", "col3"},
+				{"a,", "b", "c"},
+				{"", "e", "f"},
+				{"g", "h", "i"},
+				{"j", "", "l"},
+				{"m", "n,", ""},
+				{"p", "q", "r"},
+				{"", "", "u"},
+				{"v", "w", "x"},
+				{"y", "", ""},
+				{"", "", ""},
+			},
+			expectedDelimiter: '\t',
+		},
+		// case 2 - comma delimited with leading spaces
+		{
+			csv: ` col1,col2,col3
+ a, b, c
+d,e,f
+ ,h, i
+j, ,\x20
+ , , `,
+			expectedRows: [][]string{
+				{"col1", "col2", "col3"},
+				{"a", "b", "c"},
+				{"d", "e", "f"},
+				{"", "h", "i"},
+				{"j", "", ""},
+				{"", "", ""},
+			},
+			expectedDelimiter: ',',
+		},
+		// case 3 - every delimiter used, default to comma and handle differing number of fields per record
+		{
+			csv: `col1,col2
+a;b
+c@e
+f	g
+h|i
+jkl`,
+			expectedRows: [][]string{
+				{"col1", "col2"},
+				{"a;b"},
+				{"c@e"},
+				{"f	g"},
+				{"h|i"},
+				{"jkl"},
+			},
+			expectedDelimiter: ',',
+		},
+	}
+
+	for n, c := range cases {
+		rd, err := CreateReaderAndDetermineDelimiter(nil, strings.NewReader(decodeSlashes(t, c.csv)))
+		assert.NoError(t, err, "case %d: should not throw error: %v\n", n, err)
+		assert.Equal(t, c.expectedDelimiter, rd.Comma, "case %d: delimiter should be '%c', got '%c'", n, c.expectedDelimiter, rd.Comma)
+		rows, err := rd.ReadAll()
+		assert.NoError(t, err, "case %d: should not throw error: %v\n", n, err)
+		assert.Equal(t, c.expectedRows, rows, "case %d: rows should be equal", n)
+	}
+}
+
+type mockReader struct{}
+
+func (r *mockReader) Read(buf []byte) (int, error) {
+	return 0, io.ErrShortBuffer
+}
+
+func TestDetermineDelimiterShortBufferError(t *testing.T) {
+	rd, err := CreateReaderAndDetermineDelimiter(nil, &mockReader{})
+	assert.Error(t, err, "CreateReaderAndDetermineDelimiter() should throw an error")
+	assert.ErrorIs(t, err, io.ErrShortBuffer)
+	assert.Nil(t, rd, "CSV reader should be mnil")
+}
+
+func TestDetermineDelimiter(t *testing.T) {
+	cases := []struct {
+		csv               string
+		filename          string
+		expectedDelimiter rune
+	}{
+		// case 0 - semicolon delmited
+		{
+			csv:               "a",
+			filename:          "test.csv",
+			expectedDelimiter: ',',
+		},
+		// case 1 - single column/row CSV
+		{
+			csv:               "a",
+			filename:          "",
+			expectedDelimiter: ',',
+		},
+		// case 2 - single column, single row CSV w/ tsv file extension (so is tabbed delimited)
+		{
+			csv:               "1,2",
+			filename:          "test.tsv",
+			expectedDelimiter: '\t',
+		},
+		// case 3 - two column, single row CSV w/ no filename, so will guess comma as delimiter
+		{
+			csv:               "1,2",
+			filename:          "",
+			expectedDelimiter: ',',
+		},
+		// case 4 - semi-colon delimited with csv extension
+		{
+			csv:               "1;2",
+			filename:          "test.csv",
+			expectedDelimiter: ';',
+		},
+		// case 5 - tabbed delimited with tsv extension
+		{
+			csv:               "1\t2",
+			filename:          "test.tsv",
+			expectedDelimiter: '\t',
+		},
+		// case 6 - tabbed delimited without any filename
+		{
+			csv:               "1\t2",
+			filename:          "",
+			expectedDelimiter: '\t',
+		},
+		// case 7 - tabs won't work, only commas as every row has same amount of commas
+		{
+			csv:               "col1,col2\nfirst\tval,seconed\tval",
+			filename:          "",
+			expectedDelimiter: ',',
+		},
+		// case 8 - While looks like comma delimited, has psv extension
+		{
+			csv:               "1,2",
+			filename:          "test.psv",
+			expectedDelimiter: '|',
+		},
+		// case 9 - pipe delmiited with no extension
+		{
+			csv:               "1|2",
+			filename:          "",
+			expectedDelimiter: '|',
+		},
+		// case 10 - semi-colon delimited with commas in values
+		{
+			csv:               "1,2,3;4,5,6;7,8,9\na;b;c",
+			filename:          "",
+			expectedDelimiter: ';',
+		},
+		// case 11 - semi-colon delimited with newline in content
+		{
+			csv: `"1,2,3,4";"a
+b";%
+c;d;#`,
+			filename:          "",
+			expectedDelimiter: ';',
+		},
+		// case 12 - HTML as single value
+		{
+			csv:               "<br/>",
+			filename:          "",
+			expectedDelimiter: ',',
+		},
+		// case 13 - tab delimited with commas in values
+		{
+			csv: `name	email	note
+John Doe	john@doe.com	This,note,had,a,lot,of,commas,to,test,delimiters`,
+			filename:          "",
+			expectedDelimiter: '\t',
+		},
+	}
+
+	for n, c := range cases {
+		delimiter := determineDelimiter(markup.NewRenderContext(t.Context()).WithRelativePath(c.filename), []byte(decodeSlashes(t, c.csv)))
+		assert.Equal(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter)
+	}
+}
+
+func TestRemoveQuotedString(t *testing.T) {
+	cases := []struct {
+		text         string
+		expectedText string
+	}{
+		// case 0 - quoted text with escaped quotes in 1st column
+		{
+			text: `col1,col2,col3
+"quoted ""text"" with
+new lines
+in first column",b,c`,
+			expectedText: `col1,col2,col3
+,b,c`,
+		},
+		// case 1 - quoted text with escaped quotes in 2nd column
+		{
+			text: `col1,col2,col3
+a,"quoted ""text"" with
+new lines
+in second column",c`,
+			expectedText: `col1,col2,col3
+a,,c`,
+		},
+		// case 2 - quoted text with escaped quotes in last column
+		{
+			text: `col1,col2,col3
+a,b,"quoted ""text"" with
+new lines
+in last column"`,
+			expectedText: `col1,col2,col3
+a,b,`,
+		},
+		// case 3 - csv with lots of quotes
+		{
+			text: `a,"b",c,d,"e
+e
+e",f
+a,bb,c,d,ee ,"f
+f"
+a,b,"c ""
+c",d,e,f`,
+			expectedText: `a,,c,d,,f
+a,bb,c,d,ee ,
+a,b,,d,e,f`,
+		},
+		// case 4 - csv with pipes and quotes
+		{
+			text: `Col1 | Col2 | Col3
+abc   | "Hello
+World"|123
+"de
+
+f" | 4.56 | 789`,
+			expectedText: `Col1 | Col2 | Col3
+abc   | |123
+ | 4.56 | 789`,
+		},
+	}
+
+	for n, c := range cases {
+		modifiedText := removeQuotedString(decodeSlashes(t, c.text))
+		assert.Equal(t, c.expectedText, modifiedText, "case %d: modified text should be equal", n)
+	}
+}
+
+func TestGuessDelimiter(t *testing.T) {
+	cases := []struct {
+		csv               string
+		expectedDelimiter rune
+	}{
+		// case 0 - single cell, comma delmited
+		{
+			csv:               "a",
+			expectedDelimiter: ',',
+		},
+		// case 1 - two cells, comma delimited
+		{
+			csv:               "1,2",
+			expectedDelimiter: ',',
+		},
+		// case 2 - semicolon delimited
+		{
+			csv:               "1;2",
+			expectedDelimiter: ';',
+		},
+		// case 3 - tab delimited
+		{
+			csv:               "1\t2",
+			expectedDelimiter: '\t',
+		},
+		// case 4 - pipe delimited
+		{
+			csv:               "1|2",
+			expectedDelimiter: '|',
+		},
+		// case 5 - semicolon delimited with commas in text
+		{
+			csv: `1,2,3;4,5,6;7,8,9
+a;b;c`,
+			expectedDelimiter: ';',
+		},
+		// case 6 - semicolon delmited with commas in quoted text
+		{
+			csv: `"1,2,3,4";"a
+b"
+c;d`,
+			expectedDelimiter: ';',
+		},
+		// case 7 - HTML
+		{
+			csv:               "<br/>",
+			expectedDelimiter: ',',
+		},
+		// case 8 - tab delimited with commas in value
+		{
+			csv: `name	email	note
+John Doe	john@doe.com	This,note,had,a,lot,of,commas,to,test,delimiters`,
+			expectedDelimiter: '\t',
+		},
+		// case 9 - tab delimited with new lines in values, commas in values
+		{
+			csv: `1	"some,""more
+""
+	quoted,
+text,"	a
+2	"some,
+quoted,\t
+	text,"	b
+3	"some,
+quoted,
+	text"	c
+4	"some,
+quoted,
+text,"	d`,
+			expectedDelimiter: '\t',
+		},
+		// case 10 - semicolon delmited with quotes and semicolon in value
+		{
+			csv: `col1;col2
+"this has a literal "" in the text";"and an ; in the text"`,
+			expectedDelimiter: ';',
+		},
+		// case 11 - pipe delimited with quotes
+		{
+			csv: `Col1 | Col2 | Col3
+abc   | "Hello
+World"|123
+"de
+|
+f" | 4.56 | 789`,
+			expectedDelimiter: '|',
+		},
+		// case 12 - a tab delimited 6 column CSV, but the values are not quoted and have lots of commas.
+		// In the previous bestScore algorithm, this would have picked comma as the delimiter, but now it should guess tab
+		{
+			csv: `c1	c2	c3	c4	c5	c6
+v,k,x,v	ym,f,oa,qn,uqijh,n,s,wvygpo	uj,kt,j,w,i,fvv,tm,f,ddt,b,mwt,e,t,teq,rd,p,a	e,wfuae,t,h,q,im,ix,y	h,mrlu,l,dz,ff,zi,af,emh	,gov,bmfelvb,axp,f,u,i,cni,x,z,v,sh,w,jo,,m,h
+k,ohf,pgr,tde,m,s	te,ek,,v,,ic,kqc,dv,w,oi,j,w,gojjr,ug,,l,j,zl	g,qziq,bcajx,zfow,ka,j,re,ohbc	k,nzm,qm,ts,auf	th,elb,lx,l,q,e,qf	asbr,z,k,y,tltobga
+g,m,bu,el	h,l,jwi,o,wge,fy,rure,c,g,lcxu,fxte,uns,cl,s,o,t,h,rsoy,f	bq,s,uov,z,ikkhgyg,,sabs,c,hzue	mc,b,,j,t,n	sp,mn,,m,t,dysi,eq,pigb,rfa,z	w,rfli,sg,,o,wjjjf,f,wxdzfk,x,t,p,zy,p,mg,r,l,h
+e,ewbkc,nugd,jj,sf,ih,i,n,jo,b,poem,kw,q,i,x,t,e,uug,k	j,xm,sch,ux,h,,fb,f,pq,,mh,,f,v,,oba,w,h,v,eiz,yzd,o,a,c,e,dhp,q	a,pbef,epc,k,rdpuw,cw	k,j,e,d	xf,dz,sviv,w,sqnzew,t,b	v,yg,f,cq,ti,g,m,ta,hm,ym,ii,hxy,p,z,r,e,ga,sfs,r,p,l,aar,w,kox,j
+l,d,v,pp,q,j,bxip,w,i,im,qa,o	e,o	h,w,a,a,qzj,nt,qfn,ut,fvhu,ts	hu,q,g,p,q,ofpje,fsqa,frp,p,vih,j,w,k,jx,	ln,th,ka,l,b,vgk,rv,hkx	rj,v,y,cwm,rao,e,l,wvr,ptc,lm,yg,u,k,i,b,zk,b,gv,fls
+velxtnhlyuysbnlchosqlhkozkdapjaueexjwrndwb	nglvnv	kqiv	pbshwlmcexdzipopxjyrxhvjalwp	pydvipwlkkpdvbtepahskwuornbsb	qwbacgq
+l,y,u,bf,y,m,eals,n,cop,h,g,vs,jga,opt	x,b,zwmn,hh,b,n,pdj,t,d	px	yn,vtd,u,y,b,ps,yo,qqnem,mxg,m,al,rd,c,k,d,q,f	ilxdxa,m,y,,p,p,y,prgmg,q,n,etj,k,ns	b,pl,z,jq,hk
+p,gc	jn,mzr,bw	sb,e,r,dy,ur,wzy,r,c,n,yglr,jbdu,r,pqk,k	q,d,,,p,l,euhl,dc,rwh,t,tq,z,h,p,s,t,x,fugr,h	wi,zxb,jcig,o,t,k	mfh,ym,h,e,p,cnvx,uv,zx,x,pq,blt,v,r,u,tr,g,g,xt
+nri,p,,t,if,,y,ptlqq	a,i	w,ovli,um,w,f,re,k,sb,w,jy,zf	i,g,p,q,mii,nr,jm,cc	i,szl,k,eg,l,d	,ah,w,b,vh
+,,sh,wx,mn,xm,u,d,yy,u,t,m,j,s,b	ogadq,g,y,y,i,h,ln,jda,g,cz,s,rv,r,s,s,le,r,	y,nu,f,nagj	o,h,,adfy,o,nf,ns,gvsvnub,k,b,xyz	v,h,g,ef,y,gb	c,x,cw,x,go,h,t,x,cu,u,qgrqzrcmn,kq,cd,g,rejp,zcq
+skxg,t,vay,d,wug,d,xg,sexc	rt	g,ag,mjq,fjnyji,iwa,m,ml,b,ua,b,qjxeoc	be,s,sh,n,jbzxs,g,n,i,h,y,r,be,mfo,u,p	cw,r,,u,zn,eg,r,yac,m,l,edkr,ha,x,g,b,c,tg,c	j,ye,u,ejd,maj,ea,bm,u,iy`,
+			expectedDelimiter: '\t',
+		},
+		// case 13 - a CSV with more than 10 lines and since we only use the first 10 lines, it should still get the delimiter as semicolon
+		{
+			csv: `col1;col2;col3
+1;1;1
+2;2;2
+3;3;3
+4;4;4
+5;5;5
+6;6;6
+7;7;7
+8;8;8
+9;9;9
+10;10;10
+11	11	11
+12|12|12`,
+			expectedDelimiter: ';',
+		},
+		// case 14 - a really long single line (over 10k) that will get truncated, but since it has commas and semicolons (but more semicolons) it will pick semicolon
+		{
+			csv:               strings.Repeat("a;b,c;", 1700),
+			expectedDelimiter: ';',
+		},
+		// case 15 - 2 lines that are well over 10k, but since the 2nd line is where this CSV will be truncated (10k sample), it will only use the first line, so semicolon will be picked
+		{
+			csv:               "col1@col2@col3\na@b@" + strings.Repeat("c", 6000) + "\nd,e," + strings.Repeat("f", 4000),
+			expectedDelimiter: '@',
+		},
+		// case 16 - has all delimiters so should return comma
+		{
+			csv: `col1,col2;col3@col4|col5	col6
+a	b|c@d;e,f`,
+			expectedDelimiter: ',',
+		},
+		// case 16 - nothing works (bad csv) so returns comma by default
+		{
+			csv: `col1,col2
+a;b
+c@e
+f	g
+h|i
+jkl`,
+			expectedDelimiter: ',',
+		},
+	}
+
+	for n, c := range cases {
+		delimiter := guessDelimiter([]byte(decodeSlashes(t, c.csv)))
+		assert.Equal(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter)
+	}
+}
+
+func TestGuessFromBeforeAfterQuotes(t *testing.T) {
+	cases := []struct {
+		csv               string
+		expectedDelimiter rune
+	}{
+		// case 0 - tab delimited with new lines in values, commas in values
+		{
+			csv: `1	"some,""more
+""
+	quoted,
+text,"	a
+2	"some,
+quoted,\t
+	text,"	b
+3	"some,
+quoted,
+	text"	c
+4	"some,
+quoted,
+text,"	d`,
+			expectedDelimiter: '\t',
+		},
+		// case 1 - semicolon delmited with quotes and semicolon in value
+		{
+			csv: `col1;col2
+"this has a literal "" in the text";"and an ; in the text"`,
+			expectedDelimiter: ';',
+		},
+		// case 2 - pipe delimited with quotes
+		{
+			csv: `Col1 | Col2 | Col3
+abc   | "Hello
+World"|123
+"de
+|
+f" | 4.56 | 789`,
+			expectedDelimiter: '|',
+		},
+		// case 3 - a complicated quoted CSV that is semicolon delmiited
+		{
+			csv: `he; she
+"he said, ""hey!"""; "she said, ""hey back!"""
+but; "be"`,
+			expectedDelimiter: ';',
+		},
+		// case 4 - no delimiter should be found
+		{
+			csv:               `a,b`,
+			expectedDelimiter: 0,
+		},
+		// case 5 - no limiter should be found
+		{
+			csv: `col1
+"he said, ""here I am"""`,
+			expectedDelimiter: 0,
+		},
+		// case 6 - delimiter before double quoted string with space
+		{
+			csv: `col1|col2
+a| "he said, ""here I am"""`,
+			expectedDelimiter: '|',
+		},
+		// case 7 - delimiter before double quoted string without space
+		{
+			csv: `col1|col2
+a|"he said, ""here I am"""`,
+			expectedDelimiter: '|',
+		},
+		// case 8 - delimiter after double quoted string with space
+		{
+			csv: `col1, col2
+"abc\n
+
+", def`,
+			expectedDelimiter: ',',
+		},
+		// case 9 - delimiter after double quoted string without space
+		{
+			csv: `col1,col2
+"abc\n
+
+",def`,
+			expectedDelimiter: ',',
+		},
+	}
+
+	for n, c := range cases {
+		delimiter := guessFromBeforeAfterQuotes([]byte(decodeSlashes(t, c.csv)))
+		assert.Equal(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter)
+	}
+}
+
+func TestFormatError(t *testing.T) {
+	cases := []struct {
+		err             error
+		expectedMessage string
+		expectsError    bool
+	}{
+		{
+			err: &csv.ParseError{
+				Err: csv.ErrFieldCount,
+			},
+			expectedMessage: "repo.error.csv.invalid_field_count:0",
+			expectsError:    false,
+		},
+		{
+			err: &csv.ParseError{
+				Err: csv.ErrBareQuote,
+			},
+			expectedMessage: "repo.error.csv.unexpected:0,0",
+			expectsError:    false,
+		},
+		{
+			err:          bytes.ErrTooLarge,
+			expectsError: true,
+		},
+	}
+
+	for n, c := range cases {
+		message, err := FormatError(c.err, &translation.MockLocale{})
+		if c.expectsError {
+			assert.Error(t, err, "case %d: expected an error to be returned", n)
+		} else {
+			assert.NoError(t, err, "case %d: no error was expected, got error: %v", n, err)
+			assert.Equal(t, c.expectedMessage, message, "case %d: messages should be equal, expected '%s' got '%s'", n, c.expectedMessage, message)
+		}
+	}
+}

modules/dump/dumper.go

@@ -0,0 +1,262 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package dump
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"github.com/mholt/archives"
+)
+
+var SupportedOutputTypes = []string{"zip", "tar", "tar.sz", "tar.gz", "tar.xz", "tar.bz2", "tar.br", "tar.lz4", "tar.zst"}
+
+// PrepareFileNameAndType prepares the output file name and type, if the type is not supported, it returns an empty "outType"
+func PrepareFileNameAndType(argFile, argType string) (outFileName, outType string) {
+	if argFile == "" && argType == "" {
+		outType = SupportedOutputTypes[0]
+		outFileName = fmt.Sprintf("gitea-dump-%d.%s", timeutil.TimeStampNow(), outType)
+	} else if argFile == "" {
+		outType = argType
+		outFileName = fmt.Sprintf("gitea-dump-%d.%s", timeutil.TimeStampNow(), outType)
+	} else if argType == "" {
+		if filepath.Ext(outFileName) == "" {
+			outType = SupportedOutputTypes[0]
+			outFileName = argFile
+		} else {
+			for _, t := range SupportedOutputTypes {
+				if strings.HasSuffix(argFile, "."+t) {
+					outFileName = argFile
+					outType = t
+				}
+			}
+		}
+	} else {
+		outFileName, outType = argFile, argType
+	}
+	if !slices.Contains(SupportedOutputTypes, outType) {
+		return "", ""
+	}
+	return outFileName, outType
+}
+
+func IsSubdir(upper, lower string) (bool, error) {
+	if relPath, err := filepath.Rel(upper, lower); err != nil {
+		return false, err
+	} else if relPath == "." || !strings.HasPrefix(relPath, ".") {
+		return true, nil
+	}
+	return false, nil
+}
+
+type Dumper struct {
+	Verbose bool
+
+	jobs            chan archives.ArchiveAsyncJob
+	errArchiveAsync chan error
+	errArchiveJob   chan error
+
+	globalExcludeAbsPaths []string
+}
+
+func NewDumper(ctx context.Context, format string, output io.Writer) (*Dumper, error) {
+	d := &Dumper{
+		jobs:            make(chan archives.ArchiveAsyncJob, 1),
+		errArchiveAsync: make(chan error, 1),
+		errArchiveJob:   make(chan error, 1),
+	}
+
+	// TODO: in the future, we could completely drop the "mholt/archives" dependency.
+	// Then we only need to support "zip" and ".tar.gz" natively, and let users provide custom command line tools
+	// like "zstd" or "xz" with compression-level arguments.
+	var comp archives.ArchiverAsync
+	switch format {
+	case "zip":
+		comp = archives.Zip{}
+	case "tar":
+		comp = archives.Tar{}
+	case "tar.sz":
+		comp = archives.CompressedArchive{Compression: archives.Sz{}, Archival: archives.Tar{}}
+	case "tar.gz":
+		comp = archives.CompressedArchive{Compression: archives.Gz{}, Archival: archives.Tar{}}
+	case "tar.xz":
+		comp = archives.CompressedArchive{Compression: archives.Xz{}, Archival: archives.Tar{}}
+	case "tar.bz2":
+		comp = archives.CompressedArchive{Compression: archives.Bz2{}, Archival: archives.Tar{}}
+	case "tar.br":
+		comp = archives.CompressedArchive{Compression: archives.Brotli{}, Archival: archives.Tar{}}
+	case "tar.lz4":
+		comp = archives.CompressedArchive{Compression: archives.Lz4{}, Archival: archives.Tar{}}
+	case "tar.zst":
+		comp = archives.CompressedArchive{Compression: archives.Zstd{}, Archival: archives.Tar{}}
+	default:
+		return nil, fmt.Errorf("unsupported format: %s", format)
+	}
+	go func() {
+		d.errArchiveAsync <- comp.ArchiveAsync(ctx, output, d.jobs)
+		close(d.errArchiveAsync)
+	}()
+	return d, nil
+}
+
+func (dumper *Dumper) runArchiveJob(job archives.ArchiveAsyncJob) error {
+	dumper.jobs <- job
+	select {
+	case err := <-dumper.errArchiveAsync:
+		if err == nil {
+			return errors.New("archiver has been closed")
+		}
+		return err
+	case err := <-dumper.errArchiveJob:
+		return err
+	}
+}
+
+// AddFileByPath adds a file by its filesystem path
+func (dumper *Dumper) AddFileByPath(filePath, absPath string) error {
+	if dumper.Verbose {
+		log.Info("Adding local file %s", filePath)
+	}
+
+	fileInfo, err := os.Stat(absPath)
+	if err != nil {
+		return err
+	}
+
+	archiveFileInfo := archives.FileInfo{
+		FileInfo:      fileInfo,
+		NameInArchive: filePath,
+		Open:          func() (fs.File, error) { return os.Open(absPath) },
+	}
+
+	return dumper.runArchiveJob(archives.ArchiveAsyncJob{
+		File:   archiveFileInfo,
+		Result: dumper.errArchiveJob,
+	})
+}
+
+type readerFile struct {
+	r    io.Reader
+	info os.FileInfo
+}
+
+var _ fs.File = (*readerFile)(nil)
+
+func (f *readerFile) Stat() (fs.FileInfo, error)     { return f.info, nil }
+func (f *readerFile) Read(bytes []byte) (int, error) { return f.r.Read(bytes) }
+func (f *readerFile) Close() error                   { return nil }
+
+// AddFileByReader adds a file's contents from a Reader
+func (dumper *Dumper) AddFileByReader(r io.Reader, info os.FileInfo, customName string) error {
+	if dumper.Verbose {
+		log.Info("Adding storage file %s", customName)
+	}
+
+	fileInfo := archives.FileInfo{
+		FileInfo:      info,
+		NameInArchive: customName,
+		Open:          func() (fs.File, error) { return &readerFile{r, info}, nil },
+	}
+	return dumper.runArchiveJob(archives.ArchiveAsyncJob{
+		File:   fileInfo,
+		Result: dumper.errArchiveJob,
+	})
+}
+
+func (dumper *Dumper) Close() error {
+	close(dumper.jobs)
+	return <-dumper.errArchiveAsync
+}
+
+func (dumper *Dumper) normalizeFilePath(absPath string) string {
+	absPath = filepath.Clean(absPath)
+	if setting.IsWindows {
+		absPath = strings.ToLower(absPath)
+	}
+	return absPath
+}
+
+func (dumper *Dumper) GlobalExcludeAbsPath(absPaths ...string) {
+	for _, absPath := range absPaths {
+		dumper.globalExcludeAbsPaths = append(dumper.globalExcludeAbsPaths, dumper.normalizeFilePath(absPath))
+	}
+}
+
+func (dumper *Dumper) shouldExclude(absPath string, excludes []string) bool {
+	norm := dumper.normalizeFilePath(absPath)
+	return slices.Contains(dumper.globalExcludeAbsPaths, norm) || slices.Contains(excludes, norm)
+}
+
+func (dumper *Dumper) AddRecursiveExclude(insidePath, absPath string, excludes []string) error {
+	excludes = slices.Clone(excludes)
+	for i := range excludes {
+		excludes[i] = dumper.normalizeFilePath(excludes[i])
+	}
+	return dumper.addFileOrDir(insidePath, absPath, excludes)
+}
+
+func (dumper *Dumper) addFileOrDir(insidePath, absPath string, excludes []string) error {
+	absPath, err := filepath.Abs(absPath)
+	if err != nil {
+		return err
+	}
+	dir, err := os.Open(absPath)
+	if err != nil {
+		return err
+	}
+	defer dir.Close()
+
+	files, err := dir.Readdir(0)
+	if err != nil {
+		return err
+	}
+	for _, file := range files {
+		currentAbsPath := filepath.Join(absPath, file.Name())
+		if dumper.shouldExclude(currentAbsPath, excludes) {
+			continue
+		}
+
+		currentInsidePath := path.Join(insidePath, file.Name())
+		if file.IsDir() {
+			if err := dumper.AddFileByPath(currentInsidePath, currentAbsPath); err != nil {
+				return err
+			}
+			if err = dumper.addFileOrDir(currentInsidePath, currentAbsPath, excludes); err != nil {
+				return err
+			}
+		} else {
+			// only copy regular files and symlink regular files, skip non-regular files like socket/pipe/...
+			shouldAdd := file.Mode().IsRegular()
+			if !shouldAdd && file.Mode()&os.ModeSymlink == os.ModeSymlink {
+				target, err := filepath.EvalSymlinks(currentAbsPath)
+				if err != nil {
+					return err
+				}
+				targetStat, err := os.Stat(target)
+				if err != nil {
+					return err
+				}
+				shouldAdd = targetStat.Mode().IsRegular()
+			}
+			if shouldAdd {
+				if err = dumper.AddFileByPath(currentInsidePath, currentAbsPath); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}