fs.go 46 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711
  1. package fasthttp
  2. import (
  3. "bytes"
  4. "errors"
  5. "fmt"
  6. "html"
  7. "io"
  8. "io/fs"
  9. "mime"
  10. "net/http"
  11. "os"
  12. "path/filepath"
  13. "sort"
  14. "strings"
  15. "sync"
  16. "time"
  17. "github.com/andybalholm/brotli"
  18. "github.com/klauspost/compress/gzip"
  19. "github.com/valyala/bytebufferpool"
  20. )
  21. // ServeFileBytesUncompressed returns HTTP response containing file contents
  22. // from the given path.
  23. //
  24. // Directory contents is returned if path points to directory.
  25. //
  26. // ServeFileBytes may be used for saving network traffic when serving files
  27. // with good compression ratio.
  28. //
  29. // See also RequestCtx.SendFileBytes.
  30. //
  31. // WARNING: do not pass any user supplied paths to this function!
  32. // WARNING: if path is based on user input users will be able to request
  33. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  34. func ServeFileBytesUncompressed(ctx *RequestCtx, path []byte) {
  35. ServeFileUncompressed(ctx, b2s(path))
  36. }
  37. // ServeFileUncompressed returns HTTP response containing file contents
  38. // from the given path.
  39. //
  40. // Directory contents is returned if path points to directory.
  41. //
  42. // ServeFile may be used for saving network traffic when serving files
  43. // with good compression ratio.
  44. //
  45. // See also RequestCtx.SendFile.
  46. //
  47. // WARNING: do not pass any user supplied paths to this function!
  48. // WARNING: if path is based on user input users will be able to request
  49. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  50. func ServeFileUncompressed(ctx *RequestCtx, path string) {
  51. ctx.Request.Header.DelBytes(strAcceptEncoding)
  52. ServeFile(ctx, path)
  53. }
  54. // ServeFileBytes returns HTTP response containing compressed file contents
  55. // from the given path.
  56. //
  57. // HTTP response may contain uncompressed file contents in the following cases:
  58. //
  59. // - Missing 'Accept-Encoding: gzip' request header.
  60. // - No write access to directory containing the file.
  61. //
  62. // Directory contents is returned if path points to directory.
  63. //
  64. // Use ServeFileBytesUncompressed is you don't need serving compressed
  65. // file contents.
  66. //
  67. // See also RequestCtx.SendFileBytes.
  68. //
  69. // WARNING: do not pass any user supplied paths to this function!
  70. // WARNING: if path is based on user input users will be able to request
  71. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  72. func ServeFileBytes(ctx *RequestCtx, path []byte) {
  73. ServeFile(ctx, b2s(path))
  74. }
  75. // ServeFile returns HTTP response containing compressed file contents
  76. // from the given path.
  77. //
  78. // HTTP response may contain uncompressed file contents in the following cases:
  79. //
  80. // - Missing 'Accept-Encoding: gzip' request header.
  81. // - No write access to directory containing the file.
  82. //
  83. // Directory contents is returned if path points to directory.
  84. //
  85. // Use ServeFileUncompressed is you don't need serving compressed file contents.
  86. //
  87. // See also RequestCtx.SendFile.
  88. //
  89. // WARNING: do not pass any user supplied paths to this function!
  90. // WARNING: if path is based on user input users will be able to request
  91. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  92. func ServeFile(ctx *RequestCtx, path string) {
  93. rootFSOnce.Do(func() {
  94. rootFSHandler = rootFS.NewRequestHandler()
  95. })
  96. if path == "" || !filepath.IsAbs(path) {
  97. // extend relative path to absolute path
  98. hasTrailingSlash := len(path) > 0 && (path[len(path)-1] == '/' || path[len(path)-1] == '\\')
  99. var err error
  100. path = filepath.FromSlash(path)
  101. if path, err = filepath.Abs(path); err != nil {
  102. ctx.Logger().Printf("cannot resolve path %q to absolute file path: %v", path, err)
  103. ctx.Error("Internal Server Error", StatusInternalServerError)
  104. return
  105. }
  106. if hasTrailingSlash {
  107. path += "/"
  108. }
  109. }
  110. // convert the path to forward slashes regardless the OS in order to set the URI properly
  111. // the handler will convert back to OS path separator before opening the file
  112. path = filepath.ToSlash(path)
  113. ctx.Request.SetRequestURI(path)
  114. rootFSHandler(ctx)
  115. }
  116. var (
  117. rootFSOnce sync.Once
  118. rootFS = &FS{
  119. Root: "",
  120. AllowEmptyRoot: true,
  121. GenerateIndexPages: true,
  122. Compress: true,
  123. CompressBrotli: true,
  124. AcceptByteRange: true,
  125. }
  126. rootFSHandler RequestHandler
  127. )
  128. // ServeFS returns HTTP response containing compressed file contents from the given fs.FS's path.
  129. //
  130. // HTTP response may contain uncompressed file contents in the following cases:
  131. //
  132. // - Missing 'Accept-Encoding: gzip' request header.
  133. // - No write access to directory containing the file.
  134. //
  135. // Directory contents is returned if path points to directory.
  136. //
  137. // See also ServeFile.
  138. func ServeFS(ctx *RequestCtx, filesystem fs.FS, path string) {
  139. f := &FS{
  140. FS: filesystem,
  141. Root: "",
  142. AllowEmptyRoot: true,
  143. GenerateIndexPages: true,
  144. Compress: true,
  145. CompressBrotli: true,
  146. AcceptByteRange: true,
  147. }
  148. handler := f.NewRequestHandler()
  149. ctx.Request.SetRequestURI(path)
  150. handler(ctx)
  151. }
  152. // PathRewriteFunc must return new request path based on arbitrary ctx
  153. // info such as ctx.Path().
  154. //
  155. // Path rewriter is used in FS for translating the current request
  156. // to the local filesystem path relative to FS.Root.
  157. //
  158. // The returned path must not contain '/../' substrings due to security reasons,
  159. // since such paths may refer files outside FS.Root.
  160. //
  161. // The returned path may refer to ctx members. For example, ctx.Path().
  162. type PathRewriteFunc func(ctx *RequestCtx) []byte
  163. // NewVHostPathRewriter returns path rewriter, which strips slashesCount
  164. // leading slashes from the path and prepends the path with request's host,
  165. // thus simplifying virtual hosting for static files.
  166. //
  167. // Examples:
  168. //
  169. // - host=foobar.com, slashesCount=0, original path="/foo/bar".
  170. // Resulting path: "/foobar.com/foo/bar"
  171. //
  172. // - host=img.aaa.com, slashesCount=1, original path="/images/123/456.jpg"
  173. // Resulting path: "/img.aaa.com/123/456.jpg"
  174. func NewVHostPathRewriter(slashesCount int) PathRewriteFunc {
  175. return func(ctx *RequestCtx) []byte {
  176. path := stripLeadingSlashes(ctx.Path(), slashesCount)
  177. host := ctx.Host()
  178. if n := bytes.IndexByte(host, '/'); n >= 0 {
  179. host = nil
  180. }
  181. if len(host) == 0 {
  182. host = strInvalidHost
  183. }
  184. b := bytebufferpool.Get()
  185. b.B = append(b.B, '/')
  186. b.B = append(b.B, host...)
  187. b.B = append(b.B, path...)
  188. ctx.URI().SetPathBytes(b.B)
  189. bytebufferpool.Put(b)
  190. return ctx.Path()
  191. }
  192. }
  193. var strInvalidHost = []byte("invalid-host")
  194. // NewPathSlashesStripper returns path rewriter, which strips slashesCount
  195. // leading slashes from the path.
  196. //
  197. // Examples:
  198. //
  199. // - slashesCount = 0, original path: "/foo/bar", result: "/foo/bar"
  200. // - slashesCount = 1, original path: "/foo/bar", result: "/bar"
  201. // - slashesCount = 2, original path: "/foo/bar", result: ""
  202. //
  203. // The returned path rewriter may be used as FS.PathRewrite .
  204. func NewPathSlashesStripper(slashesCount int) PathRewriteFunc {
  205. return func(ctx *RequestCtx) []byte {
  206. return stripLeadingSlashes(ctx.Path(), slashesCount)
  207. }
  208. }
  209. // NewPathPrefixStripper returns path rewriter, which removes prefixSize bytes
  210. // from the path prefix.
  211. //
  212. // Examples:
  213. //
  214. // - prefixSize = 0, original path: "/foo/bar", result: "/foo/bar"
  215. // - prefixSize = 3, original path: "/foo/bar", result: "o/bar"
  216. // - prefixSize = 7, original path: "/foo/bar", result: "r"
  217. //
  218. // The returned path rewriter may be used as FS.PathRewrite .
  219. func NewPathPrefixStripper(prefixSize int) PathRewriteFunc {
  220. return func(ctx *RequestCtx) []byte {
  221. path := ctx.Path()
  222. if len(path) >= prefixSize {
  223. path = path[prefixSize:]
  224. }
  225. return path
  226. }
  227. }
  228. // FS represents settings for request handler serving static files
  229. // from the local filesystem.
  230. //
  231. // It is prohibited copying FS values. Create new values instead.
  232. type FS struct {
  233. noCopy noCopy
  234. // FS is filesystem to serve files from. eg: embed.FS os.DirFS
  235. FS fs.FS
  236. // Path to the root directory to serve files from.
  237. Root string
  238. // AllowEmptyRoot controls what happens when Root is empty. When false (default) it will default to the
  239. // current working directory. An empty root is mostly useful when you want to use absolute paths
  240. // on windows that are on different filesystems. On linux setting your Root to "/" already allows you to use
  241. // absolute paths on any filesystem.
  242. AllowEmptyRoot bool
  243. // List of index file names to try opening during directory access.
  244. //
  245. // For example:
  246. //
  247. // * index.html
  248. // * index.htm
  249. // * my-super-index.xml
  250. //
  251. // By default the list is empty.
  252. IndexNames []string
  253. // Index pages for directories without files matching IndexNames
  254. // are automatically generated if set.
  255. //
  256. // Directory index generation may be quite slow for directories
  257. // with many files (more than 1K), so it is discouraged enabling
  258. // index pages' generation for such directories.
  259. //
  260. // By default index pages aren't generated.
  261. GenerateIndexPages bool
  262. // Transparently compresses responses if set to true.
  263. //
  264. // The server tries minimizing CPU usage by caching compressed files.
  265. // It adds CompressedFileSuffix suffix to the original file name and
  266. // tries saving the resulting compressed file under the new file name.
  267. // So it is advisable to give the server write access to Root
  268. // and to all inner folders in order to minimize CPU usage when serving
  269. // compressed responses.
  270. //
  271. // Transparent compression is disabled by default.
  272. Compress bool
  273. // Uses brotli encoding and fallbacks to gzip in responses if set to true, uses gzip if set to false.
  274. //
  275. // This value has sense only if Compress is set.
  276. //
  277. // Brotli encoding is disabled by default.
  278. CompressBrotli bool
  279. // Path to the compressed root directory to serve files from. If this value
  280. // is empty, Root is used.
  281. CompressRoot string
  282. // Enables byte range requests if set to true.
  283. //
  284. // Byte range requests are disabled by default.
  285. AcceptByteRange bool
  286. // Path rewriting function.
  287. //
  288. // By default request path is not modified.
  289. PathRewrite PathRewriteFunc
  290. // PathNotFound fires when file is not found in filesystem
  291. // this functions tries to replace "Cannot open requested path"
  292. // server response giving to the programmer the control of server flow.
  293. //
  294. // By default PathNotFound returns
  295. // "Cannot open requested path"
  296. PathNotFound RequestHandler
  297. // SkipCache if true, will cache no file handler.
  298. //
  299. // By default is false.
  300. SkipCache bool
  301. // Expiration duration for inactive file handlers.
  302. //
  303. // FSHandlerCacheDuration is used by default.
  304. CacheDuration time.Duration
  305. // Suffix to add to the name of cached compressed file.
  306. //
  307. // This value has sense only if Compress is set.
  308. //
  309. // FSCompressedFileSuffix is used by default.
  310. CompressedFileSuffix string
  311. // Suffixes list to add to compressedFileSuffix depending on encoding
  312. //
  313. // This value has sense only if Compress is set.
  314. //
  315. // FSCompressedFileSuffixes is used by default.
  316. CompressedFileSuffixes map[string]string
  317. // If CleanStop is set, the channel can be closed to stop the cleanup handlers
  318. // for the FS RequestHandlers created with NewRequestHandler.
  319. // NEVER close this channel while the handler is still being used!
  320. CleanStop chan struct{}
  321. once sync.Once
  322. h RequestHandler
  323. }
  324. // FSCompressedFileSuffix is the suffix FS adds to the original file names
  325. // when trying to store compressed file under the new file name.
  326. // See FS.Compress for details.
  327. const FSCompressedFileSuffix = ".fasthttp.gz"
  328. // FSCompressedFileSuffixes is the suffixes FS adds to the original file names depending on encoding
  329. // when trying to store compressed file under the new file name.
  330. // See FS.Compress for details.
  331. var FSCompressedFileSuffixes = map[string]string{
  332. "gzip": ".fasthttp.gz",
  333. "br": ".fasthttp.br",
  334. }
  335. // FSHandlerCacheDuration is the default expiration duration for inactive
  336. // file handlers opened by FS.
  337. const FSHandlerCacheDuration = 10 * time.Second
  338. // FSHandler returns request handler serving static files from
  339. // the given root folder.
  340. //
  341. // stripSlashes indicates how many leading slashes must be stripped
  342. // from requested path before searching requested file in the root folder.
  343. // Examples:
  344. //
  345. // - stripSlashes = 0, original path: "/foo/bar", result: "/foo/bar"
  346. // - stripSlashes = 1, original path: "/foo/bar", result: "/bar"
  347. // - stripSlashes = 2, original path: "/foo/bar", result: ""
  348. //
  349. // The returned request handler automatically generates index pages
  350. // for directories without index.html.
  351. //
  352. // The returned handler caches requested file handles
  353. // for FSHandlerCacheDuration.
  354. // Make sure your program has enough 'max open files' limit aka
  355. // 'ulimit -n' if root folder contains many files.
  356. //
  357. // Do not create multiple request handler instances for the same
  358. // (root, stripSlashes) arguments - just reuse a single instance.
  359. // Otherwise goroutine leak will occur.
  360. func FSHandler(root string, stripSlashes int) RequestHandler {
  361. fs := &FS{
  362. Root: root,
  363. IndexNames: []string{"index.html"},
  364. GenerateIndexPages: true,
  365. AcceptByteRange: true,
  366. }
  367. if stripSlashes > 0 {
  368. fs.PathRewrite = NewPathSlashesStripper(stripSlashes)
  369. }
  370. return fs.NewRequestHandler()
  371. }
  372. // NewRequestHandler returns new request handler with the given FS settings.
  373. //
  374. // The returned handler caches requested file handles
  375. // for FS.CacheDuration.
  376. // Make sure your program has enough 'max open files' limit aka
  377. // 'ulimit -n' if FS.Root folder contains many files.
  378. //
  379. // Do not create multiple request handlers from a single FS instance -
  380. // just reuse a single request handler.
  381. func (fs *FS) NewRequestHandler() RequestHandler {
  382. fs.once.Do(fs.initRequestHandler)
  383. return fs.h
  384. }
  385. func (fs *FS) normalizeRoot(root string) string {
  386. // fs.FS uses relative paths, that paths are slash-separated on all systems, even Windows.
  387. if fs.FS == nil {
  388. // Serve files from the current working directory if Root is empty or if Root is a relative path.
  389. if (!fs.AllowEmptyRoot && root == "") || (root != "" && !filepath.IsAbs(root)) {
  390. path, err := os.Getwd()
  391. if err != nil {
  392. path = "."
  393. }
  394. root = path + "/" + root
  395. }
  396. // convert the root directory slashes to the native format
  397. root = filepath.FromSlash(root)
  398. }
  399. // strip trailing slashes from the root path
  400. for len(root) > 0 && root[len(root)-1] == os.PathSeparator {
  401. root = root[:len(root)-1]
  402. }
  403. return root
  404. }
  405. func (fs *FS) initRequestHandler() {
  406. root := fs.normalizeRoot(fs.Root)
  407. compressRoot := fs.CompressRoot
  408. if compressRoot == "" {
  409. compressRoot = root
  410. } else {
  411. compressRoot = fs.normalizeRoot(compressRoot)
  412. }
  413. compressedFileSuffixes := fs.CompressedFileSuffixes
  414. if compressedFileSuffixes["br"] == "" || compressedFileSuffixes["gzip"] == "" ||
  415. compressedFileSuffixes["br"] == compressedFileSuffixes["gzip"] {
  416. // Copy global map
  417. compressedFileSuffixes = make(map[string]string, len(FSCompressedFileSuffixes))
  418. for k, v := range FSCompressedFileSuffixes {
  419. compressedFileSuffixes[k] = v
  420. }
  421. }
  422. if len(fs.CompressedFileSuffix) > 0 {
  423. compressedFileSuffixes["gzip"] = fs.CompressedFileSuffix
  424. compressedFileSuffixes["br"] = FSCompressedFileSuffixes["br"]
  425. }
  426. h := &fsHandler{
  427. filesystem: fs.FS,
  428. root: root,
  429. indexNames: fs.IndexNames,
  430. pathRewrite: fs.PathRewrite,
  431. generateIndexPages: fs.GenerateIndexPages,
  432. compress: fs.Compress,
  433. compressBrotli: fs.CompressBrotli,
  434. compressRoot: compressRoot,
  435. pathNotFound: fs.PathNotFound,
  436. acceptByteRange: fs.AcceptByteRange,
  437. compressedFileSuffixes: compressedFileSuffixes,
  438. }
  439. h.cacheManager = newCacheManager(fs)
  440. if h.filesystem == nil {
  441. h.filesystem = &osFS{} // It provides os.Open and os.Stat
  442. }
  443. fs.h = h.handleRequest
  444. }
  445. type fsHandler struct {
  446. filesystem fs.FS
  447. root string
  448. indexNames []string
  449. pathRewrite PathRewriteFunc
  450. pathNotFound RequestHandler
  451. generateIndexPages bool
  452. compress bool
  453. compressBrotli bool
  454. compressRoot string
  455. acceptByteRange bool
  456. compressedFileSuffixes map[string]string
  457. cacheManager cacheManager
  458. smallFileReaderPool sync.Pool
  459. }
  460. type fsFile struct {
  461. h *fsHandler
  462. f fs.File
  463. filename string // fs.FileInfo.Name() return filename, isn't filepath.
  464. dirIndex []byte
  465. contentType string
  466. contentLength int
  467. compressed bool
  468. lastModified time.Time
  469. lastModifiedStr []byte
  470. t time.Time
  471. readersCount int
  472. bigFiles []*bigFileReader
  473. bigFilesLock sync.Mutex
  474. }
  475. func (ff *fsFile) NewReader() (io.Reader, error) {
  476. if ff.isBig() {
  477. r, err := ff.bigFileReader()
  478. if err != nil {
  479. ff.decReadersCount()
  480. }
  481. return r, err
  482. }
  483. return ff.smallFileReader()
  484. }
  485. func (ff *fsFile) smallFileReader() (io.Reader, error) {
  486. v := ff.h.smallFileReaderPool.Get()
  487. if v == nil {
  488. v = &fsSmallFileReader{}
  489. }
  490. r := v.(*fsSmallFileReader)
  491. r.ff = ff
  492. r.endPos = ff.contentLength
  493. if r.startPos > 0 {
  494. return nil, errors.New("bug: fsSmallFileReader with non-nil startPos found in the pool")
  495. }
  496. return r, nil
  497. }
  498. // Files bigger than this size are sent with sendfile.
  499. const maxSmallFileSize = 2 * 4096
  500. func (ff *fsFile) isBig() bool {
  501. if _, ok := ff.h.filesystem.(*osFS); !ok { // fs.FS only uses bigFileReader, memory cache uses fsSmallFileReader
  502. return ff.f != nil
  503. }
  504. return ff.contentLength > maxSmallFileSize && len(ff.dirIndex) == 0
  505. }
  506. func (ff *fsFile) bigFileReader() (io.Reader, error) {
  507. if ff.f == nil {
  508. return nil, errors.New("bug: ff.f must be non-nil in bigFileReader")
  509. }
  510. var r io.Reader
  511. ff.bigFilesLock.Lock()
  512. n := len(ff.bigFiles)
  513. if n > 0 {
  514. r = ff.bigFiles[n-1]
  515. ff.bigFiles = ff.bigFiles[:n-1]
  516. }
  517. ff.bigFilesLock.Unlock()
  518. if r != nil {
  519. return r, nil
  520. }
  521. f, err := ff.h.filesystem.Open(ff.filename)
  522. if err != nil {
  523. return nil, fmt.Errorf("cannot open already opened file: %w", err)
  524. }
  525. return &bigFileReader{
  526. f: f,
  527. ff: ff,
  528. r: f,
  529. }, nil
  530. }
  531. func (ff *fsFile) Release() {
  532. if ff.f != nil {
  533. _ = ff.f.Close()
  534. if ff.isBig() {
  535. ff.bigFilesLock.Lock()
  536. for _, r := range ff.bigFiles {
  537. _ = r.f.Close()
  538. }
  539. ff.bigFilesLock.Unlock()
  540. }
  541. }
  542. }
  543. func (ff *fsFile) decReadersCount() {
  544. ff.h.cacheManager.WithLock(func() {
  545. ff.readersCount--
  546. if ff.readersCount < 0 {
  547. ff.readersCount = 0
  548. }
  549. })
  550. }
  551. // bigFileReader attempts to trigger sendfile
  552. // for sending big files over the wire.
  553. type bigFileReader struct {
  554. f fs.File
  555. ff *fsFile
  556. r io.Reader
  557. lr io.LimitedReader
  558. }
  559. func (r *bigFileReader) UpdateByteRange(startPos, endPos int) error {
  560. seeker, ok := r.f.(io.Seeker)
  561. if !ok {
  562. return errors.New("must implement io.Seeker")
  563. }
  564. if _, err := seeker.Seek(int64(startPos), io.SeekStart); err != nil {
  565. return err
  566. }
  567. r.r = &r.lr
  568. r.lr.R = r.f
  569. r.lr.N = int64(endPos - startPos + 1)
  570. return nil
  571. }
  572. func (r *bigFileReader) Read(p []byte) (int, error) {
  573. return r.r.Read(p)
  574. }
  575. func (r *bigFileReader) WriteTo(w io.Writer) (int64, error) {
  576. if rf, ok := w.(io.ReaderFrom); ok {
  577. // fast path. Send file must be triggered
  578. return rf.ReadFrom(r.r)
  579. }
  580. // slow path
  581. return copyZeroAlloc(w, r.r)
  582. }
  583. func (r *bigFileReader) Close() error {
  584. r.r = r.f
  585. seeker, ok := r.f.(io.Seeker)
  586. if !ok {
  587. _ = r.f.Close()
  588. return errors.New("must implement io.Seeker")
  589. }
  590. n, err := seeker.Seek(0, io.SeekStart)
  591. if err == nil {
  592. if n == 0 {
  593. ff := r.ff
  594. ff.bigFilesLock.Lock()
  595. ff.bigFiles = append(ff.bigFiles, r)
  596. ff.bigFilesLock.Unlock()
  597. } else {
  598. _ = r.f.Close()
  599. err = errors.New("bug: File.Seek(0, io.SeekStart) returned (non-zero, nil)")
  600. }
  601. } else {
  602. _ = r.f.Close()
  603. }
  604. r.ff.decReadersCount()
  605. return err
  606. }
  607. type fsSmallFileReader struct {
  608. ff *fsFile
  609. startPos int
  610. endPos int
  611. }
  612. func (r *fsSmallFileReader) Close() error {
  613. ff := r.ff
  614. ff.decReadersCount()
  615. r.ff = nil
  616. r.startPos = 0
  617. r.endPos = 0
  618. ff.h.smallFileReaderPool.Put(r)
  619. return nil
  620. }
  621. func (r *fsSmallFileReader) UpdateByteRange(startPos, endPos int) error {
  622. r.startPos = startPos
  623. r.endPos = endPos + 1
  624. return nil
  625. }
  626. func (r *fsSmallFileReader) Read(p []byte) (int, error) {
  627. tailLen := r.endPos - r.startPos
  628. if tailLen <= 0 {
  629. return 0, io.EOF
  630. }
  631. if len(p) > tailLen {
  632. p = p[:tailLen]
  633. }
  634. ff := r.ff
  635. if ff.f != nil {
  636. ra, ok := ff.f.(io.ReaderAt)
  637. if !ok {
  638. return 0, errors.New("must implement io.ReaderAt")
  639. }
  640. n, err := ra.ReadAt(p, int64(r.startPos))
  641. r.startPos += n
  642. return n, err
  643. }
  644. n := copy(p, ff.dirIndex[r.startPos:])
  645. r.startPos += n
  646. return n, nil
  647. }
  648. func (r *fsSmallFileReader) WriteTo(w io.Writer) (int64, error) {
  649. ff := r.ff
  650. var n int
  651. var err error
  652. if ff.f == nil {
  653. n, err = w.Write(ff.dirIndex[r.startPos:r.endPos])
  654. return int64(n), err
  655. }
  656. if rf, ok := w.(io.ReaderFrom); ok {
  657. return rf.ReadFrom(r)
  658. }
  659. curPos := r.startPos
  660. bufv := copyBufPool.Get()
  661. buf := bufv.([]byte)
  662. for err == nil {
  663. tailLen := r.endPos - curPos
  664. if tailLen <= 0 {
  665. break
  666. }
  667. if len(buf) > tailLen {
  668. buf = buf[:tailLen]
  669. }
  670. ra, ok := ff.f.(io.ReaderAt)
  671. if !ok {
  672. return 0, errors.New("must implement io.ReaderAt")
  673. }
  674. n, err = ra.ReadAt(buf, int64(curPos))
  675. nw, errw := w.Write(buf[:n])
  676. curPos += nw
  677. if errw == nil && nw != n {
  678. errw = errors.New("bug: Write(p) returned (n, nil), where n != len(p)")
  679. }
  680. if err == nil {
  681. err = errw
  682. }
  683. }
  684. copyBufPool.Put(bufv)
  685. if err == io.EOF {
  686. err = nil
  687. }
  688. return int64(curPos - r.startPos), err
  689. }
  690. type cacheManager interface {
  691. WithLock(work func())
  692. GetFileFromCache(cacheKind CacheKind, path string) (*fsFile, bool)
  693. SetFileToCache(cacheKind CacheKind, path string, ff *fsFile) *fsFile
  694. }
  695. var (
  696. _ cacheManager = (*inMemoryCacheManager)(nil)
  697. _ cacheManager = (*noopCacheManager)(nil)
  698. )
  699. type CacheKind uint8
  700. const (
  701. defaultCacheKind CacheKind = iota
  702. brotliCacheKind
  703. gzipCacheKind
  704. )
  705. func newCacheManager(fs *FS) cacheManager {
  706. if fs.SkipCache {
  707. return &noopCacheManager{}
  708. }
  709. cacheDuration := fs.CacheDuration
  710. if cacheDuration <= 0 {
  711. cacheDuration = FSHandlerCacheDuration
  712. }
  713. instance := &inMemoryCacheManager{
  714. cacheDuration: cacheDuration,
  715. cache: make(map[string]*fsFile),
  716. cacheBrotli: make(map[string]*fsFile),
  717. cacheGzip: make(map[string]*fsFile),
  718. }
  719. go instance.handleCleanCache(fs.CleanStop)
  720. return instance
  721. }
  722. type noopCacheManager struct {
  723. cacheLock sync.Mutex
  724. }
  725. func (n *noopCacheManager) WithLock(work func()) {
  726. n.cacheLock.Lock()
  727. work()
  728. n.cacheLock.Unlock()
  729. }
  730. func (*noopCacheManager) GetFileFromCache(cacheKind CacheKind, path string) (*fsFile, bool) {
  731. return nil, false
  732. }
  733. func (*noopCacheManager) SetFileToCache(cacheKind CacheKind, path string, ff *fsFile) *fsFile {
  734. return ff
  735. }
  736. type inMemoryCacheManager struct {
  737. cacheDuration time.Duration
  738. cache map[string]*fsFile
  739. cacheBrotli map[string]*fsFile
  740. cacheGzip map[string]*fsFile
  741. cacheLock sync.Mutex
  742. }
  743. func (cm *inMemoryCacheManager) WithLock(work func()) {
  744. cm.cacheLock.Lock()
  745. work()
  746. cm.cacheLock.Unlock()
  747. }
  748. func (cm *inMemoryCacheManager) getFsCache(cacheKind CacheKind) map[string]*fsFile {
  749. fileCache := cm.cache
  750. switch cacheKind {
  751. case brotliCacheKind:
  752. fileCache = cm.cacheBrotli
  753. case gzipCacheKind:
  754. fileCache = cm.cacheGzip
  755. }
  756. return fileCache
  757. }
  758. func (cm *inMemoryCacheManager) GetFileFromCache(cacheKind CacheKind, path string) (*fsFile, bool) {
  759. fileCache := cm.getFsCache(cacheKind)
  760. cm.cacheLock.Lock()
  761. ff, ok := fileCache[path]
  762. if ok {
  763. ff.readersCount++
  764. }
  765. cm.cacheLock.Unlock()
  766. return ff, ok
  767. }
  768. func (cm *inMemoryCacheManager) SetFileToCache(cacheKind CacheKind, path string, ff *fsFile) *fsFile {
  769. fileCache := cm.getFsCache(cacheKind)
  770. cm.cacheLock.Lock()
  771. ff1, ok := fileCache[path]
  772. if !ok {
  773. fileCache[path] = ff
  774. ff.readersCount++
  775. } else {
  776. ff1.readersCount++
  777. }
  778. cm.cacheLock.Unlock()
  779. if ok {
  780. // The file has been already opened by another
  781. // goroutine, so close the current file and use
  782. // the file opened by another goroutine instead.
  783. ff.Release()
  784. ff = ff1
  785. }
  786. return ff
  787. }
  788. func (cm *inMemoryCacheManager) handleCleanCache(cleanStop chan struct{}) {
  789. var pendingFiles []*fsFile
  790. clean := func() {
  791. pendingFiles = cm.cleanCache(pendingFiles)
  792. }
  793. if cleanStop != nil {
  794. t := time.NewTicker(cm.cacheDuration / 2)
  795. for {
  796. select {
  797. case <-t.C:
  798. clean()
  799. case _, stillOpen := <-cleanStop:
  800. // Ignore values send on the channel, only stop when it is closed.
  801. if !stillOpen {
  802. t.Stop()
  803. return
  804. }
  805. }
  806. }
  807. }
  808. for {
  809. time.Sleep(cm.cacheDuration / 2)
  810. clean()
  811. }
  812. }
  813. func (cm *inMemoryCacheManager) cleanCache(pendingFiles []*fsFile) []*fsFile {
  814. var filesToRelease []*fsFile
  815. cm.cacheLock.Lock()
  816. // Close files which couldn't be closed before due to non-zero
  817. // readers count on the previous run.
  818. var remainingFiles []*fsFile
  819. for _, ff := range pendingFiles {
  820. if ff.readersCount > 0 {
  821. remainingFiles = append(remainingFiles, ff)
  822. } else {
  823. filesToRelease = append(filesToRelease, ff)
  824. }
  825. }
  826. pendingFiles = remainingFiles
  827. pendingFiles, filesToRelease = cleanCacheNolock(cm.cache, pendingFiles, filesToRelease, cm.cacheDuration)
  828. pendingFiles, filesToRelease = cleanCacheNolock(cm.cacheBrotli, pendingFiles, filesToRelease, cm.cacheDuration)
  829. pendingFiles, filesToRelease = cleanCacheNolock(cm.cacheGzip, pendingFiles, filesToRelease, cm.cacheDuration)
  830. cm.cacheLock.Unlock()
  831. for _, ff := range filesToRelease {
  832. ff.Release()
  833. }
  834. return pendingFiles
  835. }
  836. func cleanCacheNolock(
  837. cache map[string]*fsFile, pendingFiles, filesToRelease []*fsFile, cacheDuration time.Duration,
  838. ) ([]*fsFile, []*fsFile) {
  839. t := time.Now()
  840. for k, ff := range cache {
  841. if t.Sub(ff.t) > cacheDuration {
  842. if ff.readersCount > 0 {
  843. // There are pending readers on stale file handle,
  844. // so we cannot close it. Put it into pendingFiles
  845. // so it will be closed later.
  846. pendingFiles = append(pendingFiles, ff)
  847. } else {
  848. filesToRelease = append(filesToRelease, ff)
  849. }
  850. delete(cache, k)
  851. }
  852. }
  853. return pendingFiles, filesToRelease
  854. }
  855. func (h *fsHandler) pathToFilePath(path string) string {
  856. if _, ok := h.filesystem.(*osFS); !ok {
  857. if len(path) < 1 {
  858. return path
  859. }
  860. return path[1:]
  861. }
  862. return filepath.FromSlash(h.root + path)
  863. }
  864. func (h *fsHandler) filePathToCompressed(filePath string) string {
  865. if h.root == h.compressRoot {
  866. return filePath
  867. }
  868. if !strings.HasPrefix(filePath, h.root) {
  869. return filePath
  870. }
  871. return filepath.FromSlash(h.compressRoot + filePath[len(h.root):])
  872. }
  873. func (h *fsHandler) handleRequest(ctx *RequestCtx) {
  874. var path []byte
  875. if h.pathRewrite != nil {
  876. path = h.pathRewrite(ctx)
  877. } else {
  878. path = ctx.Path()
  879. }
  880. hasTrailingSlash := len(path) > 0 && path[len(path)-1] == '/'
  881. path = stripTrailingSlashes(path)
  882. if n := bytes.IndexByte(path, 0); n >= 0 {
  883. ctx.Logger().Printf("cannot serve path with nil byte at position %d: %q", n, path)
  884. ctx.Error("Are you a hacker?", StatusBadRequest)
  885. return
  886. }
  887. if h.pathRewrite != nil {
  888. // There is no need to check for '/../' if path = ctx.Path(),
  889. // since ctx.Path must normalize and sanitize the path.
  890. if n := bytes.Index(path, strSlashDotDotSlash); n >= 0 {
  891. ctx.Logger().Printf("cannot serve path with '/../' at position %d due to security reasons: %q", n, path)
  892. ctx.Error("Internal Server Error", StatusInternalServerError)
  893. return
  894. }
  895. }
  896. mustCompress := false
  897. fileCacheKind := defaultCacheKind
  898. fileEncoding := ""
  899. byteRange := ctx.Request.Header.peek(strRange)
  900. if len(byteRange) == 0 && h.compress {
  901. if h.compressBrotli && ctx.Request.Header.HasAcceptEncodingBytes(strBr) {
  902. mustCompress = true
  903. fileCacheKind = brotliCacheKind
  904. fileEncoding = "br"
  905. } else if ctx.Request.Header.HasAcceptEncodingBytes(strGzip) {
  906. mustCompress = true
  907. fileCacheKind = gzipCacheKind
  908. fileEncoding = "gzip"
  909. }
  910. }
  911. pathStr := string(path)
  912. ff, ok := h.cacheManager.GetFileFromCache(fileCacheKind, pathStr)
  913. if !ok {
  914. filePath := h.pathToFilePath(pathStr)
  915. var err error
  916. ff, err = h.openFSFile(filePath, mustCompress, fileEncoding)
  917. if mustCompress && err == errNoCreatePermission {
  918. ctx.Logger().Printf("insufficient permissions for saving compressed file for %q. Serving uncompressed file. "+
  919. "Allow write access to the directory with this file in order to improve fasthttp performance", filePath)
  920. mustCompress = false
  921. ff, err = h.openFSFile(filePath, mustCompress, fileEncoding)
  922. }
  923. if err == errDirIndexRequired {
  924. if !hasTrailingSlash {
  925. ctx.RedirectBytes(append(path, '/'), StatusFound)
  926. return
  927. }
  928. ff, err = h.openIndexFile(ctx, filePath, mustCompress, fileEncoding)
  929. if err != nil {
  930. ctx.Logger().Printf("cannot open dir index %q: %v", filePath, err)
  931. ctx.Error("Directory index is forbidden", StatusForbidden)
  932. return
  933. }
  934. } else if err != nil {
  935. ctx.Logger().Printf("cannot open file %q: %v", filePath, err)
  936. if h.pathNotFound == nil {
  937. ctx.Error("Cannot open requested path", StatusNotFound)
  938. } else {
  939. ctx.SetStatusCode(StatusNotFound)
  940. h.pathNotFound(ctx)
  941. }
  942. return
  943. }
  944. ff = h.cacheManager.SetFileToCache(fileCacheKind, pathStr, ff)
  945. }
  946. if !ctx.IfModifiedSince(ff.lastModified) {
  947. ff.decReadersCount()
  948. ctx.NotModified()
  949. return
  950. }
  951. r, err := ff.NewReader()
  952. if err != nil {
  953. ctx.Logger().Printf("cannot obtain file reader for path=%q: %v", path, err)
  954. ctx.Error("Internal Server Error", StatusInternalServerError)
  955. return
  956. }
  957. hdr := &ctx.Response.Header
  958. if ff.compressed {
  959. if fileEncoding == "br" {
  960. hdr.SetContentEncodingBytes(strBr)
  961. } else if fileEncoding == "gzip" {
  962. hdr.SetContentEncodingBytes(strGzip)
  963. }
  964. }
  965. statusCode := StatusOK
  966. contentLength := ff.contentLength
  967. if h.acceptByteRange {
  968. hdr.setNonSpecial(strAcceptRanges, strBytes)
  969. if len(byteRange) > 0 {
  970. startPos, endPos, err := ParseByteRange(byteRange, contentLength)
  971. if err != nil {
  972. _ = r.(io.Closer).Close()
  973. ctx.Logger().Printf("cannot parse byte range %q for path=%q: %v", byteRange, path, err)
  974. ctx.Error("Range Not Satisfiable", StatusRequestedRangeNotSatisfiable)
  975. return
  976. }
  977. if err = r.(byteRangeUpdater).UpdateByteRange(startPos, endPos); err != nil {
  978. _ = r.(io.Closer).Close()
  979. ctx.Logger().Printf("cannot seek byte range %q for path=%q: %v", byteRange, path, err)
  980. ctx.Error("Internal Server Error", StatusInternalServerError)
  981. return
  982. }
  983. hdr.SetContentRange(startPos, endPos, contentLength)
  984. contentLength = endPos - startPos + 1
  985. statusCode = StatusPartialContent
  986. }
  987. }
  988. hdr.setNonSpecial(strLastModified, ff.lastModifiedStr)
  989. if !ctx.IsHead() {
  990. ctx.SetBodyStream(r, contentLength)
  991. } else {
  992. ctx.Response.ResetBody()
  993. ctx.Response.SkipBody = true
  994. ctx.Response.Header.SetContentLength(contentLength)
  995. if rc, ok := r.(io.Closer); ok {
  996. if err := rc.Close(); err != nil {
  997. ctx.Logger().Printf("cannot close file reader: %v", err)
  998. ctx.Error("Internal Server Error", StatusInternalServerError)
  999. return
  1000. }
  1001. }
  1002. }
  1003. hdr.noDefaultContentType = true
  1004. if len(hdr.ContentType()) == 0 {
  1005. ctx.SetContentType(ff.contentType)
  1006. }
  1007. ctx.SetStatusCode(statusCode)
  1008. }
  1009. type byteRangeUpdater interface {
  1010. UpdateByteRange(startPos, endPos int) error
  1011. }
  1012. // ParseByteRange parses 'Range: bytes=...' header value.
  1013. //
  1014. // It follows https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 .
  1015. func ParseByteRange(byteRange []byte, contentLength int) (startPos, endPos int, err error) {
  1016. b := byteRange
  1017. if !bytes.HasPrefix(b, strBytes) {
  1018. return 0, 0, fmt.Errorf("unsupported range units: %q. Expecting %q", byteRange, strBytes)
  1019. }
  1020. b = b[len(strBytes):]
  1021. if len(b) == 0 || b[0] != '=' {
  1022. return 0, 0, fmt.Errorf("missing byte range in %q", byteRange)
  1023. }
  1024. b = b[1:]
  1025. n := bytes.IndexByte(b, '-')
  1026. if n < 0 {
  1027. return 0, 0, fmt.Errorf("missing the end position of byte range in %q", byteRange)
  1028. }
  1029. if n == 0 {
  1030. v, err := ParseUint(b[n+1:])
  1031. if err != nil {
  1032. return 0, 0, err
  1033. }
  1034. startPos := contentLength - v
  1035. if startPos < 0 {
  1036. startPos = 0
  1037. }
  1038. return startPos, contentLength - 1, nil
  1039. }
  1040. if startPos, err = ParseUint(b[:n]); err != nil {
  1041. return 0, 0, err
  1042. }
  1043. if startPos >= contentLength {
  1044. return 0, 0, fmt.Errorf("the start position of byte range cannot exceed %d. byte range %q", contentLength-1, byteRange)
  1045. }
  1046. b = b[n+1:]
  1047. if len(b) == 0 {
  1048. return startPos, contentLength - 1, nil
  1049. }
  1050. if endPos, err = ParseUint(b); err != nil {
  1051. return 0, 0, err
  1052. }
  1053. if endPos >= contentLength {
  1054. endPos = contentLength - 1
  1055. }
  1056. if endPos < startPos {
  1057. return 0, 0, fmt.Errorf("the start position of byte range cannot exceed the end position. byte range %q", byteRange)
  1058. }
  1059. return startPos, endPos, nil
  1060. }
  1061. func (h *fsHandler) openIndexFile(ctx *RequestCtx, dirPath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
  1062. for _, indexName := range h.indexNames {
  1063. indexFilePath := dirPath + "/" + indexName
  1064. ff, err := h.openFSFile(indexFilePath, mustCompress, fileEncoding)
  1065. if err == nil {
  1066. return ff, nil
  1067. }
  1068. if !errors.Is(err, fs.ErrNotExist) {
  1069. return nil, fmt.Errorf("cannot open file %q: %w", indexFilePath, err)
  1070. }
  1071. }
  1072. if !h.generateIndexPages {
  1073. return nil, fmt.Errorf("cannot access directory without index page. Directory %q", dirPath)
  1074. }
  1075. return h.createDirIndex(ctx, dirPath, mustCompress, fileEncoding)
  1076. }
  1077. var (
  1078. errDirIndexRequired = errors.New("directory index required")
  1079. errNoCreatePermission = errors.New("no 'create file' permissions")
  1080. )
  1081. func (h *fsHandler) createDirIndex(ctx *RequestCtx, dirPath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
  1082. w := &bytebufferpool.ByteBuffer{}
  1083. base := ctx.URI()
  1084. basePathEscaped := html.EscapeString(string(base.Path()))
  1085. _, _ = fmt.Fprintf(w, "<html><head><title>%s</title><style>.dir { font-weight: bold }</style></head><body>", basePathEscaped)
  1086. _, _ = fmt.Fprintf(w, "<h1>%s</h1>", basePathEscaped)
  1087. _, _ = fmt.Fprintf(w, "<ul>")
  1088. if len(basePathEscaped) > 1 {
  1089. var parentURI URI
  1090. base.CopyTo(&parentURI)
  1091. parentURI.Update(string(base.Path()) + "/..")
  1092. parentPathEscaped := html.EscapeString(string(parentURI.Path()))
  1093. _, _ = fmt.Fprintf(w, `<li><a href="%s" class="dir">..</a></li>`, parentPathEscaped)
  1094. }
  1095. dirEntries, err := fs.ReadDir(h.filesystem, dirPath)
  1096. if err != nil {
  1097. return nil, err
  1098. }
  1099. fm := make(map[string]fs.FileInfo, len(dirEntries))
  1100. filenames := make([]string, 0, len(dirEntries))
  1101. nestedContinue:
  1102. for _, de := range dirEntries {
  1103. name := de.Name()
  1104. for _, cfs := range h.compressedFileSuffixes {
  1105. if strings.HasSuffix(name, cfs) {
  1106. // Do not show compressed files on index page.
  1107. continue nestedContinue
  1108. }
  1109. }
  1110. fi, err := de.Info()
  1111. if err != nil {
  1112. ctx.Logger().Printf("cannot fetch information from dir entry %q: %v, skip", name, err)
  1113. continue nestedContinue
  1114. }
  1115. fm[name] = fi
  1116. filenames = append(filenames, name)
  1117. }
  1118. var u URI
  1119. base.CopyTo(&u)
  1120. u.Update(string(u.Path()) + "/")
  1121. sort.Strings(filenames)
  1122. for _, name := range filenames {
  1123. u.Update(name)
  1124. pathEscaped := html.EscapeString(string(u.Path()))
  1125. fi := fm[name]
  1126. auxStr := "dir"
  1127. className := "dir"
  1128. if !fi.IsDir() {
  1129. auxStr = fmt.Sprintf("file, %d bytes", fi.Size())
  1130. className = "file"
  1131. }
  1132. _, _ = fmt.Fprintf(w, `<li><a href="%s" class="%s">%s</a>, %s, last modified %s</li>`,
  1133. pathEscaped, className, html.EscapeString(name), auxStr, fsModTime(fi.ModTime()))
  1134. }
  1135. _, _ = fmt.Fprintf(w, "</ul></body></html>")
  1136. if mustCompress {
  1137. var zbuf bytebufferpool.ByteBuffer
  1138. if fileEncoding == "br" {
  1139. zbuf.B = AppendBrotliBytesLevel(zbuf.B, w.B, CompressDefaultCompression)
  1140. } else if fileEncoding == "gzip" {
  1141. zbuf.B = AppendGzipBytesLevel(zbuf.B, w.B, CompressDefaultCompression)
  1142. }
  1143. w = &zbuf
  1144. }
  1145. dirIndex := w.B
  1146. lastModified := time.Now()
  1147. ff := &fsFile{
  1148. h: h,
  1149. dirIndex: dirIndex,
  1150. contentType: "text/html; charset=utf-8",
  1151. contentLength: len(dirIndex),
  1152. compressed: mustCompress,
  1153. lastModified: lastModified,
  1154. lastModifiedStr: AppendHTTPDate(nil, lastModified),
  1155. t: lastModified,
  1156. }
  1157. return ff, nil
  1158. }
  1159. const (
  1160. fsMinCompressRatio = 0.8
  1161. fsMaxCompressibleFileSize = 8 * 1024 * 1024
  1162. )
  1163. func (h *fsHandler) compressAndOpenFSFile(filePath string, fileEncoding string) (*fsFile, error) {
  1164. f, err := h.filesystem.Open(filePath)
  1165. if err != nil {
  1166. return nil, err
  1167. }
  1168. fileInfo, err := f.Stat()
  1169. if err != nil {
  1170. _ = f.Close()
  1171. return nil, fmt.Errorf("cannot obtain info for file %q: %w", filePath, err)
  1172. }
  1173. if fileInfo.IsDir() {
  1174. _ = f.Close()
  1175. return nil, errDirIndexRequired
  1176. }
  1177. if strings.HasSuffix(filePath, h.compressedFileSuffixes[fileEncoding]) ||
  1178. fileInfo.Size() > fsMaxCompressibleFileSize ||
  1179. !isFileCompressible(f, fsMinCompressRatio) {
  1180. return h.newFSFile(f, fileInfo, false, filePath, "")
  1181. }
  1182. compressedFilePath := h.filePathToCompressed(filePath)
  1183. if _, ok := h.filesystem.(*osFS); !ok {
  1184. return h.newCompressedFSFileCache(f, fileInfo, compressedFilePath, fileEncoding)
  1185. }
  1186. if compressedFilePath != filePath {
  1187. if err := os.MkdirAll(filepath.Dir(compressedFilePath), os.ModePerm); err != nil {
  1188. return nil, err
  1189. }
  1190. }
  1191. compressedFilePath += h.compressedFileSuffixes[fileEncoding]
  1192. absPath, err := filepath.Abs(compressedFilePath)
  1193. if err != nil {
  1194. _ = f.Close()
  1195. return nil, fmt.Errorf("cannot determine absolute path for %q: %v", compressedFilePath, err)
  1196. }
  1197. flock := getFileLock(absPath)
  1198. flock.Lock()
  1199. ff, err := h.compressFileNolock(f, fileInfo, filePath, compressedFilePath, fileEncoding)
  1200. flock.Unlock()
  1201. return ff, err
  1202. }
  1203. func (h *fsHandler) compressFileNolock(
  1204. f fs.File, fileInfo fs.FileInfo, filePath, compressedFilePath, fileEncoding string,
  1205. ) (*fsFile, error) {
  1206. // Attempt to open compressed file created by another concurrent
  1207. // goroutine.
  1208. // It is safe opening such a file, since the file creation
  1209. // is guarded by file mutex - see getFileLock call.
  1210. if _, err := os.Stat(compressedFilePath); err == nil {
  1211. _ = f.Close()
  1212. return h.newCompressedFSFile(compressedFilePath, fileEncoding)
  1213. }
  1214. // Create temporary file, so concurrent goroutines don't use
  1215. // it until it is created.
  1216. tmpFilePath := compressedFilePath + ".tmp"
  1217. zf, err := os.Create(tmpFilePath)
  1218. if err != nil {
  1219. _ = f.Close()
  1220. if !errors.Is(err, fs.ErrPermission) {
  1221. return nil, fmt.Errorf("cannot create temporary file %q: %w", tmpFilePath, err)
  1222. }
  1223. return nil, errNoCreatePermission
  1224. }
  1225. if fileEncoding == "br" {
  1226. zw := acquireStacklessBrotliWriter(zf, CompressDefaultCompression)
  1227. _, err = copyZeroAlloc(zw, f)
  1228. if err1 := zw.Flush(); err == nil {
  1229. err = err1
  1230. }
  1231. releaseStacklessBrotliWriter(zw, CompressDefaultCompression)
  1232. } else if fileEncoding == "gzip" {
  1233. zw := acquireStacklessGzipWriter(zf, CompressDefaultCompression)
  1234. _, err = copyZeroAlloc(zw, f)
  1235. if err1 := zw.Flush(); err == nil {
  1236. err = err1
  1237. }
  1238. releaseStacklessGzipWriter(zw, CompressDefaultCompression)
  1239. }
  1240. _ = zf.Close()
  1241. _ = f.Close()
  1242. if err != nil {
  1243. return nil, fmt.Errorf("error when compressing file %q to %q: %w", filePath, tmpFilePath, err)
  1244. }
  1245. if err = os.Chtimes(tmpFilePath, time.Now(), fileInfo.ModTime()); err != nil {
  1246. return nil, fmt.Errorf("cannot change modification time to %v for tmp file %q: %v",
  1247. fileInfo.ModTime(), tmpFilePath, err)
  1248. }
  1249. if err = os.Rename(tmpFilePath, compressedFilePath); err != nil {
  1250. return nil, fmt.Errorf("cannot move compressed file from %q to %q: %w", tmpFilePath, compressedFilePath, err)
  1251. }
  1252. return h.newCompressedFSFile(compressedFilePath, fileEncoding)
  1253. }
  1254. // newCompressedFSFileCache use memory cache compressed files.
  1255. func (h *fsHandler) newCompressedFSFileCache(f fs.File, fileInfo fs.FileInfo, filePath, fileEncoding string) (*fsFile, error) {
  1256. var (
  1257. w = &bytebufferpool.ByteBuffer{}
  1258. err error
  1259. )
  1260. if fileEncoding == "br" {
  1261. zw := acquireStacklessBrotliWriter(w, CompressDefaultCompression)
  1262. _, err = copyZeroAlloc(zw, f)
  1263. if err1 := zw.Flush(); err == nil {
  1264. err = err1
  1265. }
  1266. releaseStacklessBrotliWriter(zw, CompressDefaultCompression)
  1267. } else if fileEncoding == "gzip" {
  1268. zw := acquireStacklessGzipWriter(w, CompressDefaultCompression)
  1269. _, err = copyZeroAlloc(zw, f)
  1270. if err1 := zw.Flush(); err == nil {
  1271. err = err1
  1272. }
  1273. releaseStacklessGzipWriter(zw, CompressDefaultCompression)
  1274. }
  1275. defer func() { _ = f.Close() }()
  1276. if err != nil {
  1277. return nil, fmt.Errorf("error when compressing file %q: %w", filePath, err)
  1278. }
  1279. seeker, ok := f.(io.Seeker)
  1280. if !ok {
  1281. return nil, errors.New("not implemented io.Seeker")
  1282. }
  1283. if _, err = seeker.Seek(0, io.SeekStart); err != nil {
  1284. return nil, err
  1285. }
  1286. ext := fileExtension(fileInfo.Name(), false, h.compressedFileSuffixes[fileEncoding])
  1287. contentType := mime.TypeByExtension(ext)
  1288. if contentType == "" {
  1289. data, err := readFileHeader(f, false, fileEncoding)
  1290. if err != nil {
  1291. return nil, fmt.Errorf("cannot read header of the file %q: %w", fileInfo.Name(), err)
  1292. }
  1293. contentType = http.DetectContentType(data)
  1294. }
  1295. dirIndex := w.B
  1296. lastModified := fileInfo.ModTime()
  1297. ff := &fsFile{
  1298. h: h,
  1299. dirIndex: dirIndex,
  1300. contentType: contentType,
  1301. contentLength: len(dirIndex),
  1302. compressed: true,
  1303. lastModified: lastModified,
  1304. lastModifiedStr: AppendHTTPDate(nil, lastModified),
  1305. t: time.Now(),
  1306. }
  1307. return ff, nil
  1308. }
  1309. func (h *fsHandler) newCompressedFSFile(filePath string, fileEncoding string) (*fsFile, error) {
  1310. f, err := h.filesystem.Open(filePath)
  1311. if err != nil {
  1312. return nil, fmt.Errorf("cannot open compressed file %q: %w", filePath, err)
  1313. }
  1314. fileInfo, err := f.Stat()
  1315. if err != nil {
  1316. _ = f.Close()
  1317. return nil, fmt.Errorf("cannot obtain info for compressed file %q: %w", filePath, err)
  1318. }
  1319. return h.newFSFile(f, fileInfo, true, filePath, fileEncoding)
  1320. }
  1321. func (h *fsHandler) openFSFile(filePath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
  1322. filePathOriginal := filePath
  1323. if mustCompress {
  1324. filePath += h.compressedFileSuffixes[fileEncoding]
  1325. }
  1326. f, err := h.filesystem.Open(filePath)
  1327. if err != nil {
  1328. if mustCompress && errors.Is(err, fs.ErrNotExist) {
  1329. return h.compressAndOpenFSFile(filePathOriginal, fileEncoding)
  1330. }
  1331. return nil, err
  1332. }
  1333. fileInfo, err := f.Stat()
  1334. if err != nil {
  1335. _ = f.Close()
  1336. return nil, fmt.Errorf("cannot obtain info for file %q: %w", filePath, err)
  1337. }
  1338. if fileInfo.IsDir() {
  1339. _ = f.Close()
  1340. if mustCompress {
  1341. return nil, fmt.Errorf("directory with unexpected suffix found: %q. Suffix: %q",
  1342. filePath, h.compressedFileSuffixes[fileEncoding])
  1343. }
  1344. return nil, errDirIndexRequired
  1345. }
  1346. if mustCompress {
  1347. fileInfoOriginal, err := fs.Stat(h.filesystem, filePathOriginal)
  1348. if err != nil {
  1349. _ = f.Close()
  1350. return nil, fmt.Errorf("cannot obtain info for original file %q: %w", filePathOriginal, err)
  1351. }
  1352. // Only re-create the compressed file if there was more than a second between the mod times.
  1353. // On macOS the gzip seems to truncate the nanoseconds in the mod time causing the original file
  1354. // to look newer than the gzipped file.
  1355. if fileInfoOriginal.ModTime().Sub(fileInfo.ModTime()) >= time.Second {
  1356. // The compressed file became stale. Re-create it.
  1357. _ = f.Close()
  1358. _ = os.Remove(filePath)
  1359. return h.compressAndOpenFSFile(filePathOriginal, fileEncoding)
  1360. }
  1361. }
  1362. return h.newFSFile(f, fileInfo, mustCompress, filePath, fileEncoding)
  1363. }
  1364. func (h *fsHandler) newFSFile(f fs.File, fileInfo fs.FileInfo, compressed bool, filePath, fileEncoding string) (*fsFile, error) {
  1365. n := fileInfo.Size()
  1366. contentLength := int(n)
  1367. if n != int64(contentLength) {
  1368. _ = f.Close()
  1369. return nil, fmt.Errorf("too big file: %d bytes", n)
  1370. }
  1371. // detect content-type
  1372. ext := fileExtension(fileInfo.Name(), compressed, h.compressedFileSuffixes[fileEncoding])
  1373. contentType := mime.TypeByExtension(ext)
  1374. if contentType == "" {
  1375. data, err := readFileHeader(f, compressed, fileEncoding)
  1376. if err != nil {
  1377. return nil, fmt.Errorf("cannot read header of the file %q: %w", fileInfo.Name(), err)
  1378. }
  1379. contentType = http.DetectContentType(data)
  1380. }
  1381. lastModified := fileInfo.ModTime()
  1382. ff := &fsFile{
  1383. h: h,
  1384. f: f,
  1385. filename: filePath,
  1386. contentType: contentType,
  1387. contentLength: contentLength,
  1388. compressed: compressed,
  1389. lastModified: lastModified,
  1390. lastModifiedStr: AppendHTTPDate(nil, lastModified),
  1391. t: time.Now(),
  1392. }
  1393. return ff, nil
  1394. }
  1395. func readFileHeader(f io.Reader, compressed bool, fileEncoding string) ([]byte, error) {
  1396. r := f
  1397. var (
  1398. br *brotli.Reader
  1399. zr *gzip.Reader
  1400. )
  1401. if compressed {
  1402. var err error
  1403. if fileEncoding == "br" {
  1404. if br, err = acquireBrotliReader(f); err != nil {
  1405. return nil, err
  1406. }
  1407. r = br
  1408. } else if fileEncoding == "gzip" {
  1409. if zr, err = acquireGzipReader(f); err != nil {
  1410. return nil, err
  1411. }
  1412. r = zr
  1413. }
  1414. }
  1415. lr := &io.LimitedReader{
  1416. R: r,
  1417. N: 512,
  1418. }
  1419. data, err := io.ReadAll(lr)
  1420. seeker, ok := f.(io.Seeker)
  1421. if !ok {
  1422. return nil, errors.New("must implement io.Seeker")
  1423. }
  1424. if _, err := seeker.Seek(0, io.SeekStart); err != nil {
  1425. return nil, err
  1426. }
  1427. if br != nil {
  1428. releaseBrotliReader(br)
  1429. }
  1430. if zr != nil {
  1431. releaseGzipReader(zr)
  1432. }
  1433. return data, err
  1434. }
  1435. func stripLeadingSlashes(path []byte, stripSlashes int) []byte {
  1436. for stripSlashes > 0 && len(path) > 0 {
  1437. if path[0] != '/' {
  1438. // developer sanity-check
  1439. panic("BUG: path must start with slash")
  1440. }
  1441. n := bytes.IndexByte(path[1:], '/')
  1442. if n < 0 {
  1443. path = path[:0]
  1444. break
  1445. }
  1446. path = path[n+1:]
  1447. stripSlashes--
  1448. }
  1449. return path
  1450. }
  1451. func stripTrailingSlashes(path []byte) []byte {
  1452. for len(path) > 0 && path[len(path)-1] == '/' {
  1453. path = path[:len(path)-1]
  1454. }
  1455. return path
  1456. }
  1457. func fileExtension(path string, compressed bool, compressedFileSuffix string) string {
  1458. if compressed && strings.HasSuffix(path, compressedFileSuffix) {
  1459. path = path[:len(path)-len(compressedFileSuffix)]
  1460. }
  1461. n := strings.LastIndexByte(path, '.')
  1462. if n < 0 {
  1463. return ""
  1464. }
  1465. return path[n:]
  1466. }
  1467. // FileLastModified returns last modified time for the file.
  1468. func FileLastModified(path string) (time.Time, error) {
  1469. f, err := os.Open(path)
  1470. if err != nil {
  1471. return zeroTime, err
  1472. }
  1473. fileInfo, err := f.Stat()
  1474. _ = f.Close()
  1475. if err != nil {
  1476. return zeroTime, err
  1477. }
  1478. return fsModTime(fileInfo.ModTime()), nil
  1479. }
  1480. func fsModTime(t time.Time) time.Time {
  1481. return t.In(time.UTC).Truncate(time.Second)
  1482. }
  1483. var filesLockMap sync.Map
  1484. func getFileLock(absPath string) *sync.Mutex {
  1485. v, _ := filesLockMap.LoadOrStore(absPath, &sync.Mutex{})
  1486. filelock := v.(*sync.Mutex)
  1487. return filelock
  1488. }
  1489. var _ fs.FS = (*osFS)(nil)
  1490. type osFS struct{}
  1491. func (o *osFS) Open(name string) (fs.File, error) { return os.Open(name) }
  1492. func (o *osFS) Stat(name string) (fs.FileInfo, error) { return os.Stat(name) }