github-forks
/
gitea
mirror of https://github.com/go-gitea/gitea.git
1
0
Fork 0
Code Projects Releases Activity

Fix SSH LFS memory usage (#33455)

Browse Source 
Fix #33448
pull/33459/head^2
wxiaoguang 12 hours ago committed by GitHub
parent 4f3cc26b4e
commit 0e8738b4b6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 77 additions and 103 deletions
Show Stats Download Patch File Download Diff File

13
modules/httplib/request.go
Unescape Escape View File

@ -99,10 +99,10 @@ func (r *Request) Param(key, value string) *Request {
return r
}
// Body adds request raw body.
// it supports string and []byte.
// Body adds request raw body. It supports string, []byte and io.Reader as body.
func (r *Request) Body(data any) *Request {
switch t := data.(type) {
case nil: // do nothing
case string:
bf := bytes.NewBufferString(t)
r.req.Body = io.NopCloser(bf)
@ -111,6 +111,12 @@ func (r *Request) Body(data any) *Request {
bf := bytes.NewBuffer(t)
r.req.Body = io.NopCloser(bf)
r.req.ContentLength = int64(len(t))
case io.ReadCloser:
r.req.Body = t
case io.Reader:
r.req.Body = io.NopCloser(t)
default:
panic(fmt.Sprintf("unsupported request body type %T", t))
}
return r
}
@ -141,7 +147,7 @@ func (r *Request) getResponse() (*http.Response, error) {
}
} else if r.req.Method == "POST" && r.req.Body == nil && len(paramBody) > 0 {
r.Header("Content-Type", "application/x-www-form-urlencoded")
r.Body(paramBody)
r.Body(paramBody) // string
}
var err error
@ -185,6 +191,7 @@ func (r *Request) getResponse() (*http.Response, error) {
}
// Response executes request client gets response manually.
// Caller MUST close the response body if no error occurs
func (r *Request) Response() (*http.Response, error) {
return r.getResponse()
}

40
modules/lfstransfer/backend/backend.go
Unescape Escape View File

@ -4,7 +4,6 @@
package backend
import (
"bytes"
"context"
"encoding/base64"
"fmt"
@ -29,7 +28,7 @@ var Capabilities = []string{
"locking",
}
var _ transfer.Backend = &GiteaBackend{}
var _ transfer.Backend = (*GiteaBackend)(nil)
// GiteaBackend is an adapter between git-lfs-transfer library and Gitea's internal LFS API
type GiteaBackend struct {
@ -78,17 +77,17 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
req := newInternalRequestLFS(g.ctx, url, http.MethodPost, headers, bodyBytes)
resp, err := req.Response()
if err != nil {
g.logger.Log("http request error", err)
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode))
return nil, statusCodeToErr(resp.StatusCode)
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
g.logger.Log("http read error", err)
@ -158,8 +157,7 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans
return pointers, nil
}
// Download implements transfer.Backend. The returned reader must be closed by the
// caller.
// Download implements transfer.Backend. The returned reader must be closed by the caller.
func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser, int64, error) {
idMapStr, exists := args[argID]
if !exists {
@ -187,25 +185,25 @@ func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser,
headerGiteaInternalAuth: g.internalAuth,
headerAccept: mimeOctetStream,
}
req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil)
req := newInternalRequestLFS(g.ctx, url, http.MethodGet, headers, nil)
resp, err := req.Response()
if err != nil {
return nil, 0, err
return nil, 0, fmt.Errorf("failed to get response: %w", err)
}
// no need to close the body here by "defer resp.Body.Close()", see below
if resp.StatusCode != http.StatusOK {
return nil, 0, statusCodeToErr(resp.StatusCode)
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
respSize, err := strconv.ParseInt(resp.Header.Get("X-Gitea-LFS-Content-Length"), 10, 64)
if err != nil {
return nil, 0, err
return nil, 0, fmt.Errorf("failed to parse content length: %w", err)
}
respSize := int64(len(respBytes))
respBuf := io.NopCloser(bytes.NewBuffer(respBytes))
return respBuf, respSize, nil
// transfer.Backend will check io.Closer interface and close this Body reader
return resp.Body, respSize, nil
}
// StartUpload implements transfer.Backend.
// Upload implements transfer.Backend.
func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, args transfer.Args) error {
idMapStr, exists := args[argID]
if !exists {
@ -234,15 +232,14 @@ func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, args transfer
headerContentType: mimeOctetStream,
headerContentLength: strconv.FormatInt(size, 10),
}
reqBytes, err := io.ReadAll(r)
if err != nil {
return err
}
req := newInternalRequest(g.ctx, url, http.MethodPut, headers, reqBytes)
req := newInternalRequestLFS(g.ctx, url, http.MethodPut, headers, nil)
req.Body(r)
resp, err := req.Response()
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return statusCodeToErr(resp.StatusCode)
}
@ -284,11 +281,12 @@ func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (trans
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
req := newInternalRequestLFS(g.ctx, url, http.MethodPost, headers, bodyBytes)
resp, err := req.Response()
if err != nil {
return transfer.NewStatus(transfer.StatusInternalServerError), err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return transfer.NewStatus(uint32(resp.StatusCode), http.StatusText(resp.StatusCode)), statusCodeToErr(resp.StatusCode)
}

6
modules/lfstransfer/backend/lock.go
Unescape Escape View File

@ -50,7 +50,7 @@ func (g *giteaLockBackend) Create(path, refname string) (transfer.Lock, error) {
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
req := newInternalRequestLFS(g.ctx, url, http.MethodPost, headers, bodyBytes)
resp, err := req.Response()
if err != nil {
g.logger.Log("http request error", err)
@ -102,7 +102,7 @@ func (g *giteaLockBackend) Unlock(lock transfer.Lock) error {
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
req := newInternalRequestLFS(g.ctx, url, http.MethodPost, headers, bodyBytes)
resp, err := req.Response()
if err != nil {
g.logger.Log("http request error", err)
@ -185,7 +185,7 @@ func (g *giteaLockBackend) queryLocks(v url.Values) ([]transfer.Lock, string, er
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil)
req := newInternalRequestLFS(g.ctx, url, http.MethodGet, headers, nil)
resp, err := req.Response()
if err != nil {
g.logger.Log("http request error", err)

63
modules/lfstransfer/backend/util.go
Unescape Escape View File

@ -5,15 +5,12 @@ package backend
import (
"context"
"crypto/tls"
"fmt"
"net"
"io"
"net/http"
"time"
"code.gitea.io/gitea/modules/httplib"
"code.gitea.io/gitea/modules/proxyprotocol"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/private"
"github.com/charmbracelet/git-lfs-transfer/transfer"
)
@ -89,53 +86,19 @@ func statusCodeToErr(code int) error {
}
}
func newInternalRequest(ctx context.Context, url, method string, headers map[string]string, body []byte) *httplib.Request {
req := httplib.NewRequest(url, method).
SetContext(ctx).
SetTimeout(10*time.Second, 60*time.Second).
SetTLSClientConfig(&tls.Config{
InsecureSkipVerify: true,
})
if setting.Protocol == setting.HTTPUnix {
req.SetTransport(&http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
var d net.Dialer
conn, err := d.DialContext(ctx, "unix", setting.HTTPAddr)
if err != nil {
return conn, err
}
if setting.LocalUseProxyProtocol {
if err = proxyprotocol.WriteLocalHeader(conn); err != nil {
_ = conn.Close()
return nil, err
}
}
return conn, err
},
})
} else if setting.LocalUseProxyProtocol {
req.SetTransport(&http.Transport{
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
var d net.Dialer
conn, err := d.DialContext(ctx, network, address)
if err != nil {
return conn, err
}
if err = proxyprotocol.WriteLocalHeader(conn); err != nil {
_ = conn.Close()
return nil, err
}
return conn, err
},
})
}
func newInternalRequestLFS(ctx context.Context, url, method string, headers map[string]string, body any) *httplib.Request {
req := private.NewInternalRequest(ctx, url, method)
for k, v := range headers {
req.Header(k, v)
}
req.Body(body)
switch body := body.(type) {
case nil: // do nothing
case []byte:
req.Body(body) // []byte
case io.Reader:
req.Body(body) // io.Reader or io.ReadCloser
default:
panic(fmt.Sprintf("unsupported request body type %T", body))
}
return req
}

2
modules/private/actions.go
Unescape Escape View File

@ -17,7 +17,7 @@ type GenerateTokenRequest struct {
func GenerateActionsRunnerToken(ctx context.Context, scope string) (*ResponseText, ResponseExtra) {
reqURL := setting.LocalURL + "api/internal/actions/generate_actions_runner_token"
req := newInternalRequest(ctx, reqURL, "POST", GenerateTokenRequest{
req := newInternalRequestAPI(ctx, reqURL, "POST", GenerateTokenRequest{
Scope: scope,
})

10
modules/private/hook.go
Unescape Escape View File

@ -85,7 +85,7 @@ type HookProcReceiveRefResult struct {
// HookPreReceive check whether the provided commits are allowed
func HookPreReceive(ctx context.Context, ownerName, repoName string, opts HookOptions) ResponseExtra {
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName))
req := newInternalRequest(ctx, reqURL, "POST", opts)
req := newInternalRequestAPI(ctx, reqURL, "POST", opts)
req.SetReadWriteTimeout(time.Duration(60+len(opts.OldCommitIDs)) * time.Second)
_, extra := requestJSONResp(req, &ResponseText{})
return extra
@ -94,7 +94,7 @@ func HookPreReceive(ctx context.Context, ownerName, repoName string, opts HookOp
// HookPostReceive updates services and users
func HookPostReceive(ctx context.Context, ownerName, repoName string, opts HookOptions) (*HookPostReceiveResult, ResponseExtra) {
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/post-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName))
req := newInternalRequest(ctx, reqURL, "POST", opts)
req := newInternalRequestAPI(ctx, reqURL, "POST", opts)
req.SetReadWriteTimeout(time.Duration(60+len(opts.OldCommitIDs)) * time.Second)
return requestJSONResp(req, &HookPostReceiveResult{})
}
@ -103,7 +103,7 @@ func HookPostReceive(ctx context.Context, ownerName, repoName string, opts HookO
func HookProcReceive(ctx context.Context, ownerName, repoName string, opts HookOptions) (*HookProcReceiveResult, ResponseExtra) {
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/proc-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName))
req := newInternalRequest(ctx, reqURL, "POST", opts)
req := newInternalRequestAPI(ctx, reqURL, "POST", opts)
req.SetReadWriteTimeout(time.Duration(60+len(opts.OldCommitIDs)) * time.Second)
return requestJSONResp(req, &HookProcReceiveResult{})
}
@ -115,7 +115,7 @@ func SetDefaultBranch(ctx context.Context, ownerName, repoName, branch string) R
url.PathEscape(repoName),
url.PathEscape(branch),
)
req := newInternalRequest(ctx, reqURL, "POST")
req := newInternalRequestAPI(ctx, reqURL, "POST")
_, extra := requestJSONResp(req, &ResponseText{})
return extra
}
@ -123,7 +123,7 @@ func SetDefaultBranch(ctx context.Context, ownerName, repoName, branch string) R
// SSHLog sends ssh error log response
func SSHLog(ctx context.Context, isErr bool, msg string) error {
reqURL := setting.LocalURL + "api/internal/ssh/log"
req := newInternalRequest(ctx, reqURL, "POST", &SSHLogOption{IsError: isErr, Message: msg})
req := newInternalRequestAPI(ctx, reqURL, "POST", &SSHLogOption{IsError: isErr, Message: msg})
_, extra := requestJSONResp(req, &ResponseText{})
return extra.Error
}

8
modules/private/internal.go
Unescape Escape View File

@ -34,7 +34,7 @@ func getClientIP() string {
return strings.Fields(sshConnEnv)[0]
}
func newInternalRequest(ctx context.Context, url, method string, body ...any) *httplib.Request {
func NewInternalRequest(ctx context.Context, url, method string) *httplib.Request {
if setting.InternalToken == "" {
log.Fatal(`The INTERNAL_TOKEN setting is missing from the configuration file: %q.
Ensure you are running in the correct environment or set the correct configuration file with -c.`, setting.CustomConf)
@ -82,13 +82,17 @@ Ensure you are running in the correct environment or set the correct configurati
},
})
}
return req
}
func newInternalRequestAPI(ctx context.Context, url, method string, body ...any) *httplib.Request {
req := NewInternalRequest(ctx, url, method)
if len(body) == 1 {
req.Header("Content-Type", "application/json")
jsonBytes, _ := json.Marshal(body[0])
req.Body(jsonBytes)
} else if len(body) > 1 {
log.Fatal("Too many arguments for newInternalRequest")
log.Fatal("Too many arguments for newInternalRequestAPI")
}
req.SetTimeout(10*time.Second, 60*time.Second)

4
modules/private/key.go
Unescape Escape View File

@ -14,7 +14,7 @@ import (
func UpdatePublicKeyInRepo(ctx context.Context, keyID, repoID int64) error {
// Ask for running deliver hook and test pull request tasks.
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/ssh/%d/update/%d", keyID, repoID)
req := newInternalRequest(ctx, reqURL, "POST")
req := newInternalRequestAPI(ctx, reqURL, "POST")
_, extra := requestJSONResp(req, &ResponseText{})
return extra.Error
}
@ -24,7 +24,7 @@ func UpdatePublicKeyInRepo(ctx context.Context, keyID, repoID int64) error {
func AuthorizedPublicKeyByContent(ctx context.Context, content string) (*ResponseText, ResponseExtra) {
// Ask for running deliver hook and test pull request tasks.
reqURL := setting.LocalURL + "api/internal/ssh/authorized_keys"
req := newInternalRequest(ctx, reqURL, "POST")
req := newInternalRequestAPI(ctx, reqURL, "POST")
req.Param("content", content)
return requestJSONResp(req, &ResponseText{})
}

2
modules/private/mail.go
Unescape Escape View File

@ -23,7 +23,7 @@ type Email struct {
func SendEmail(ctx context.Context, subject, message string, to []string) (*ResponseText, ResponseExtra) {
reqURL := setting.LocalURL + "api/internal/mail/send"
req := newInternalRequest(ctx, reqURL, "POST", Email{
req := newInternalRequestAPI(ctx, reqURL, "POST", Email{
Subject: subject,
Message: message,
To: to,

22
modules/private/manager.go
Unescape Escape View File

@ -18,21 +18,21 @@ import (
// Shutdown calls the internal shutdown function
func Shutdown(ctx context.Context) ResponseExtra {
reqURL := setting.LocalURL + "api/internal/manager/shutdown"
req := newInternalRequest(ctx, reqURL, "POST")
req := newInternalRequestAPI(ctx, reqURL, "POST")
return requestJSONClientMsg(req, "Shutting down")
}
// Restart calls the internal restart function
func Restart(ctx context.Context) ResponseExtra {
reqURL := setting.LocalURL + "api/internal/manager/restart"
req := newInternalRequest(ctx, reqURL, "POST")
req := newInternalRequestAPI(ctx, reqURL, "POST")
return requestJSONClientMsg(req, "Restarting")
}
// ReloadTemplates calls the internal reload-templates function
func ReloadTemplates(ctx context.Context) ResponseExtra {
reqURL := setting.LocalURL + "api/internal/manager/reload-templates"
req := newInternalRequest(ctx, reqURL, "POST")
req := newInternalRequestAPI(ctx, reqURL, "POST")
return requestJSONClientMsg(req, "Reloaded")
}
@ -45,7 +45,7 @@ type FlushOptions struct {
// FlushQueues calls the internal flush-queues function
func FlushQueues(ctx context.Context, timeout time.Duration, nonBlocking bool) ResponseExtra {
reqURL := setting.LocalURL + "api/internal/manager/flush-queues"
req := newInternalRequest(ctx, reqURL, "POST", FlushOptions{Timeout: timeout, NonBlocking: nonBlocking})
req := newInternalRequestAPI(ctx, reqURL, "POST", FlushOptions{Timeout: timeout, NonBlocking: nonBlocking})
if timeout > 0 {
req.SetReadWriteTimeout(timeout + 10*time.Second)
}
@ -55,28 +55,28 @@ func FlushQueues(ctx context.Context, timeout time.Duration, nonBlocking bool) R
// PauseLogging pauses logging
func PauseLogging(ctx context.Context) ResponseExtra {
reqURL := setting.LocalURL + "api/internal/manager/pause-logging"
req := newInternalRequest(ctx, reqURL, "POST")
req := newInternalRequestAPI(ctx, reqURL, "POST")
return requestJSONClientMsg(req, "Logging Paused")
}
// ResumeLogging resumes logging
func ResumeLogging(ctx context.Context) ResponseExtra {
reqURL := setting.LocalURL + "api/internal/manager/resume-logging"
req := newInternalRequest(ctx, reqURL, "POST")
req := newInternalRequestAPI(ctx, reqURL, "POST")
return requestJSONClientMsg(req, "Logging Restarted")
}
// ReleaseReopenLogging releases and reopens logging files
func ReleaseReopenLogging(ctx context.Context) ResponseExtra {
reqURL := setting.LocalURL + "api/internal/manager/release-and-reopen-logging"
req := newInternalRequest(ctx, reqURL, "POST")
req := newInternalRequestAPI(ctx, reqURL, "POST")
return requestJSONClientMsg(req, "Logging Restarted")
}
// SetLogSQL sets database logging
func SetLogSQL(ctx context.Context, on bool) ResponseExtra {
reqURL := setting.LocalURL + "api/internal/manager/set-log-sql?on=" + strconv.FormatBool(on)
req := newInternalRequest(ctx, reqURL, "POST")
req := newInternalRequestAPI(ctx, reqURL, "POST")
return requestJSONClientMsg(req, "Log SQL setting set")
}
@ -91,7 +91,7 @@ type LoggerOptions struct {
// AddLogger adds a logger
func AddLogger(ctx context.Context, logger, writer, mode string, config map[string]any) ResponseExtra {
reqURL := setting.LocalURL + "api/internal/manager/add-logger"
req := newInternalRequest(ctx, reqURL, "POST", LoggerOptions{
req := newInternalRequestAPI(ctx, reqURL, "POST", LoggerOptions{
Logger: logger,
Writer: writer,
Mode: mode,
@ -103,7 +103,7 @@ func AddLogger(ctx context.Context, logger, writer, mode string, config map[stri
// RemoveLogger removes a logger
func RemoveLogger(ctx context.Context, logger, writer string) ResponseExtra {
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/remove-logger/%s/%s", url.PathEscape(logger), url.PathEscape(writer))
req := newInternalRequest(ctx, reqURL, "POST")
req := newInternalRequestAPI(ctx, reqURL, "POST")
return requestJSONClientMsg(req, "Removed")
}
@ -111,7 +111,7 @@ func RemoveLogger(ctx context.Context, logger, writer string) ResponseExtra {
func Processes(ctx context.Context, out io.Writer, flat, noSystem, stacktraces, json bool, cancel string) ResponseExtra {
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/processes?flat=%t&no-system=%t&stacktraces=%t&json=%t&cancel-pid=%s", flat, noSystem, stacktraces, json, url.QueryEscape(cancel))
req := newInternalRequest(ctx, reqURL, "GET")
req := newInternalRequestAPI(ctx, reqURL, "GET")
callback := func(resp *http.Response, extra *ResponseExtra) {
_, extra.Error = io.Copy(out, resp.Body)
}

2
modules/private/restore_repo.go
Unescape Escape View File

@ -24,7 +24,7 @@ type RestoreParams struct {
func RestoreRepo(ctx context.Context, repoDir, ownerName, repoName string, units []string, validation bool) ResponseExtra {
reqURL := setting.LocalURL + "api/internal/restore_repo"
req := newInternalRequest(ctx, reqURL, "POST", RestoreParams{
req := newInternalRequestAPI(ctx, reqURL, "POST", RestoreParams{
RepoDir: repoDir,
OwnerName: ownerName,
RepoName: repoName,

4
modules/private/serv.go
Unescape Escape View File

@ -23,7 +23,7 @@ type KeyAndOwner struct {
// ServNoCommand returns information about the provided key
func ServNoCommand(ctx context.Context, keyID int64) (*asymkey_model.PublicKey, *user_model.User, error) {
reqURL := setting.LocalURL + fmt.Sprintf("api/internal/serv/none/%d", keyID)
req := newInternalRequest(ctx, reqURL, "GET")
req := newInternalRequestAPI(ctx, reqURL, "GET")
keyAndOwner, extra := requestJSONResp(req, &KeyAndOwner{})
if extra.HasError() {
return nil, nil, extra.Error
@ -58,6 +58,6 @@ func ServCommand(ctx context.Context, keyID int64, ownerName, repoName string, m
reqURL += fmt.Sprintf("&verb=%s", url.QueryEscape(verb))
}
}
req := newInternalRequest(ctx, reqURL, "GET")
req := newInternalRequestAPI(ctx, reqURL, "GET")
return requestJSONResp(req, &ServCommandResults{})
}

4
services/lfs/server.go
Unescape Escape View File

@ -134,7 +134,9 @@ func DownloadHandler(ctx *context.Context) {
}
contentLength := toByte + 1 - fromByte
ctx.Resp.Header().Set("Content-Length", strconv.FormatInt(contentLength, 10))
contentLengthStr := strconv.FormatInt(contentLength, 10)
ctx.Resp.Header().Set("Content-Length", contentLengthStr)
ctx.Resp.Header().Set("X-Gitea-LFS-Content-Length", contentLengthStr) // we need this header to make sure it won't be affected by reverse proxy or compression
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
filename := ctx.PathParam("filename")

Write Preview
Loading…
Cancel
Save