Skip to content

Commit df9612b

Browse files
qwerty287Gustedzeripathlunny
authored
Add API to serve blob or LFS file content (#19689)
* Add LFS API * Update routers/api/v1/repo/file.go Co-authored-by: Gusted <williamzijl7@hotmail.com> * Apply suggestions * Apply suggestions * Update routers/api/v1/repo/file.go Co-authored-by: Gusted <williamzijl7@hotmail.com> * Report errors * ADd test * Use own repo for test * Use different repo name * Improve handling * Slight restructures 1. Avoid reading the blob data multiple times 2. Ensure that caching is only checked when about to serve the blob/lfs 3. Avoid nesting by returning early 4. Make log message a bit more clear 5. Ensure that the dataRc is closed by defer when passed to ServeData Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: Gusted <williamzijl7@hotmail.com> Co-authored-by: Andrew Thornton <art27@cantab.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
1 parent 14d96ff commit df9612b

File tree

4 files changed

+245
-0
lines changed

4 files changed

+245
-0
lines changed

‎integrations/api_repo_file_get_test.go‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package integrations
6+
7+
import (
8+
"net/http"
9+
"net/url"
10+
"os"
11+
"testing"
12+
13+
api "code.gitea.io/gitea/modules/structs"
14+
"code.gitea.io/gitea/modules/util"
15+
16+
"github.com/stretchr/testify/assert"
17+
)
18+
19+
func TestAPIGetRawFileOrLFS(t *testing.T) {
20+
defer prepareTestEnv(t)()
21+
22+
// Test with raw file
23+
req := NewRequest(t, "GET", "/api/v1/repos/user2/repo1/media/README.md")
24+
resp := MakeRequest(t, req, http.StatusOK)
25+
assert.Equal(t, "# repo1\n\nDescription for repo1", resp.Body.String())
26+
27+
// Test with LFS
28+
onGiteaRun(t, func(t *testing.T, u *url.URL) {
29+
httpContext := NewAPITestContext(t, "user2", "repo-lfs-test")
30+
doAPICreateRepository(httpContext, false, func(t *testing.T, repository api.Repository) {
31+
u.Path = httpContext.GitPath()
32+
dstPath, err := os.MkdirTemp("", httpContext.Reponame)
33+
assert.NoError(t, err)
34+
defer util.RemoveAll(dstPath)
35+
36+
u.Path = httpContext.GitPath()
37+
u.User = url.UserPassword("user2", userPassword)
38+
39+
t.Run("Clone", doGitClone(dstPath, u))
40+
41+
dstPath2, err := os.MkdirTemp("", httpContext.Reponame)
42+
assert.NoError(t, err)
43+
defer util.RemoveAll(dstPath2)
44+
45+
t.Run("Partial Clone", doPartialGitClone(dstPath2, u))
46+
47+
lfs, _ := lfsCommitAndPushTest(t, dstPath)
48+
49+
reqLFS := NewRequest(t, "GET", "/api/v1/repos/user2/repo1/media/"+lfs)
50+
respLFS := MakeRequestNilResponseRecorder(t, reqLFS, http.StatusOK)
51+
assert.Equal(t, littleSize, respLFS.Length)
52+
53+
doAPIDeleteRepository(httpContext)
54+
})
55+
})
56+
}

‎routers/api/v1/api.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -826,6 +826,7 @@ func Routes() *web.Route {
826826
Delete(reqAdmin(), repo.DeleteTeam)
827827
}, reqToken())
828828
m.Get("/raw/*", context.ReferencesGitRepo(), context.RepoRefForAPI, reqRepoReader(unit.TypeCode), repo.GetRawFile)
829+
m.Get("/media/*", context.ReferencesGitRepo(), context.RepoRefForAPI, reqRepoReader(unit.TypeCode), repo.GetRawFileOrLFS)
829830
m.Get("/archive/*", reqRepoReader(unit.TypeCode), repo.GetArchive)
830831
m.Combo("/forks").Get(repo.ListForks).
831832
Post(reqToken(), reqRepoReader(unit.TypeCode), bind(api.CreateForkOption{}), repo.CreateFork)

‎routers/api/v1/repo/file.go‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@
66
package repo
77

88
import (
9+
"bytes"
910
"encoding/base64"
1011
"fmt"
12+
"io"
1113
"net/http"
1214
"path"
1315
"time"
@@ -18,7 +20,11 @@ import (
1820
"code.gitea.io/gitea/modules/cache"
1921
"code.gitea.io/gitea/modules/context"
2022
"code.gitea.io/gitea/modules/git"
23+
"code.gitea.io/gitea/modules/httpcache"
24+
"code.gitea.io/gitea/modules/lfs"
25+
"code.gitea.io/gitea/modules/log"
2126
"code.gitea.io/gitea/modules/setting"
27+
"code.gitea.io/gitea/modules/storage"
2228
api "code.gitea.io/gitea/modules/structs"
2329
"code.gitea.io/gitea/modules/web"
2430
"code.gitea.io/gitea/routers/common"
@@ -75,6 +81,142 @@ func GetRawFile(ctx *context.APIContext) {
7581
}
7682
}
7783

84+
// GetRawFileOrLFS get a file by repo's path, redirecting to LFS if necessary.
85+
func GetRawFileOrLFS(ctx *context.APIContext) {
86+
// swagger:operation GET /repos/{owner}/{repo}/media/{filepath} repository repoGetRawFileOrLFS
87+
// ---
88+
// summary: Get a file or it's LFS object from a repository
89+
// parameters:
90+
// - name: owner
91+
// in: path
92+
// description: owner of the repo
93+
// type: string
94+
// required: true
95+
// - name: repo
96+
// in: path
97+
// description: name of the repo
98+
// type: string
99+
// required: true
100+
// - name: filepath
101+
// in: path
102+
// description: filepath of the file to get
103+
// type: string
104+
// required: true
105+
// - name: ref
106+
// in: query
107+
// description: "The name of the commit/branch/tag. Default the repository’s default branch (usually master)"
108+
// type: string
109+
// required: false
110+
// responses:
111+
// 200:
112+
// description: Returns raw file content.
113+
// "404":
114+
// "$ref": "#/responses/notFound"
115+
116+
if ctx.Repo.Repository.IsEmpty {
117+
ctx.NotFound()
118+
return
119+
}
120+
121+
blob, lastModified := getBlobForEntry(ctx)
122+
if ctx.Written() {
123+
return
124+
}
125+
126+
// LFS Pointer files are at most 1024 bytes - so any blob greater than 1024 bytes cannot be an LFS file
127+
if blob.Size() > 1024 {
128+
// First handle caching for the blob
129+
if httpcache.HandleGenericETagTimeCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`, lastModified) {
130+
return
131+
}
132+
133+
// OK not cached - serve!
134+
if err := common.ServeBlob(ctx.Context, blob, lastModified); err != nil {
135+
ctx.ServerError("ServeBlob", err)
136+
}
137+
return
138+
}
139+
140+
// OK, now the blob is known to have at most 1024 bytes we can simply read this in in one go (This saves reading it twice)
141+
dataRc, err := blob.DataAsync()
142+
if err != nil {
143+
ctx.ServerError("DataAsync", err)
144+
return
145+
}
146+
147+
buf, err := io.ReadAll(dataRc)
148+
if err != nil {
149+
_ = dataRc.Close()
150+
ctx.ServerError("DataAsync", err)
151+
return
152+
}
153+
154+
if err := dataRc.Close(); err != nil {
155+
log.Error("Error whilst closing blob %s reader in %-v. Error: %v", blob.ID, ctx.Context.Repo.Repository, err)
156+
}
157+
158+
// Check if the blob represents a pointer
159+
pointer, _ := lfs.ReadPointer(bytes.NewReader(buf))
160+
161+
// if its not a pointer just serve the data directly
162+
if !pointer.IsValid() {
163+
// First handle caching for the blob
164+
if httpcache.HandleGenericETagTimeCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`, lastModified) {
165+
return
166+
}
167+
168+
// OK not cached - serve!
169+
if err := common.ServeData(ctx.Context, ctx.Repo.TreePath, blob.Size(), bytes.NewReader(buf)); err != nil {
170+
ctx.ServerError("ServeBlob", err)
171+
}
172+
return
173+
}
174+
175+
// Now check if there is a meta object for this pointer
176+
meta, err := models.GetLFSMetaObjectByOid(ctx.Repo.Repository.ID, pointer.Oid)
177+
178+
// If there isn't one just serve the data directly
179+
if err == models.ErrLFSObjectNotExist {
180+
// Handle caching for the blob SHA (not the LFS object OID)
181+
if httpcache.HandleGenericETagTimeCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`, lastModified) {
182+
return
183+
}
184+
185+
if err := common.ServeData(ctx.Context, ctx.Repo.TreePath, blob.Size(), bytes.NewReader(buf)); err != nil {
186+
ctx.ServerError("ServeBlob", err)
187+
}
188+
return
189+
} else if err != nil {
190+
ctx.ServerError("GetLFSMetaObjectByOid", err)
191+
return
192+
}
193+
194+
// Handle caching for the LFS object OID
195+
if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+pointer.Oid+`"`) {
196+
return
197+
}
198+
199+
if setting.LFS.ServeDirect {
200+
// If we have a signed url (S3, object storage), redirect to this directly.
201+
u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name())
202+
if u != nil && err == nil {
203+
ctx.Redirect(u.String())
204+
return
205+
}
206+
}
207+
208+
lfsDataRc, err := lfs.ReadMetaObject(meta.Pointer)
209+
if err != nil {
210+
ctx.ServerError("ReadMetaObject", err)
211+
return
212+
}
213+
defer lfsDataRc.Close()
214+
215+
if err := common.ServeData(ctx.Context, ctx.Repo.TreePath, meta.Size, lfsDataRc); err != nil {
216+
ctx.ServerError("ServeData", err)
217+
}
218+
}
219+
78220
func getBlobForEntry(ctx *context.APIContext) (blob *git.Blob, lastModified time.Time) {
79221
entry, err := ctx.Repo.Commit.GetTreeEntryByPath(ctx.Repo.TreePath)
80222
if err != nil {

‎templates/swagger/v1_json.tmpl‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7150,6 +7150,52 @@
71507150
}
71517151
}
71527152
},
7153+
"/repos/{owner}/{repo}/media/{filepath}": {
7154+
"get": {
7155+
"tags": [
7156+
"repository"
7157+
],
7158+
"summary": "Get a file or it's LFS object from a repository",
7159+
"operationId": "repoGetRawFileOrLFS",
7160+
"parameters": [
7161+
{
7162+
"type": "string",
7163+
"description": "owner of the repo",
7164+
"name": "owner",
7165+
"in": "path",
7166+
"required": true
7167+
},
7168+
{
7169+
"type": "string",
7170+
"description": "name of the repo",
7171+
"name": "repo",
7172+
"in": "path",
7173+
"required": true
7174+
},
7175+
{
7176+
"type": "string",
7177+
"description": "filepath of the file to get",
7178+
"name": "filepath",
7179+
"in": "path",
7180+
"required": true
7181+
},
7182+
{
7183+
"type": "string",
7184+
"description": "The name of the commit/branch/tag. Default the repository’s default branch (usually master)",
7185+
"name": "ref",
7186+
"in": "query"
7187+
}
7188+
],
7189+
"responses": {
7190+
"200": {
7191+
"description": "Returns raw file content."
7192+
},
7193+
"404": {
7194+
"$ref": "#/responses/notFound"
7195+
}
7196+
}
7197+
}
7198+
},
71537199
"/repos/{owner}/{repo}/milestones": {
71547200
"get": {
71557201
"produces": [

0 commit comments

Comments
 (0)