fs.go 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707
  1. package fasthttp
  2. import (
  3. "bytes"
  4. "errors"
  5. "fmt"
  6. "html"
  7. "io"
  8. "io/fs"
  9. "mime"
  10. "net/http"
  11. "os"
  12. "path/filepath"
  13. "sort"
  14. "strings"
  15. "sync"
  16. "time"
  17. "github.com/andybalholm/brotli"
  18. "github.com/klauspost/compress/gzip"
  19. "github.com/valyala/bytebufferpool"
  20. )
  21. // ServeFileBytesUncompressed returns HTTP response containing file contents
  22. // from the given path.
  23. //
  24. // Directory contents is returned if path points to directory.
  25. //
  26. // ServeFileBytes may be used for saving network traffic when serving files
  27. // with good compression ratio.
  28. //
  29. // See also RequestCtx.SendFileBytes.
  30. //
  31. // WARNING: do not pass any user supplied paths to this function!
  32. // WARNING: if path is based on user input users will be able to request
  33. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  34. func ServeFileBytesUncompressed(ctx *RequestCtx, path []byte) {
  35. ServeFileUncompressed(ctx, b2s(path))
  36. }
  37. // ServeFileUncompressed returns HTTP response containing file contents
  38. // from the given path.
  39. //
  40. // Directory contents is returned if path points to directory.
  41. //
  42. // ServeFile may be used for saving network traffic when serving files
  43. // with good compression ratio.
  44. //
  45. // See also RequestCtx.SendFile.
  46. //
  47. // WARNING: do not pass any user supplied paths to this function!
  48. // WARNING: if path is based on user input users will be able to request
  49. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  50. func ServeFileUncompressed(ctx *RequestCtx, path string) {
  51. ctx.Request.Header.DelBytes(strAcceptEncoding)
  52. ServeFile(ctx, path)
  53. }
  54. // ServeFileBytes returns HTTP response containing compressed file contents
  55. // from the given path.
  56. //
  57. // HTTP response may contain uncompressed file contents in the following cases:
  58. //
  59. // - Missing 'Accept-Encoding: gzip' request header.
  60. // - No write access to directory containing the file.
  61. //
  62. // Directory contents is returned if path points to directory.
  63. //
  64. // Use ServeFileBytesUncompressed is you don't need serving compressed
  65. // file contents.
  66. //
  67. // See also RequestCtx.SendFileBytes.
  68. //
  69. // WARNING: do not pass any user supplied paths to this function!
  70. // WARNING: if path is based on user input users will be able to request
  71. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  72. func ServeFileBytes(ctx *RequestCtx, path []byte) {
  73. ServeFile(ctx, b2s(path))
  74. }
  75. // ServeFile returns HTTP response containing compressed file contents
  76. // from the given path.
  77. //
  78. // HTTP response may contain uncompressed file contents in the following cases:
  79. //
  80. // - Missing 'Accept-Encoding: gzip' request header.
  81. // - No write access to directory containing the file.
  82. //
  83. // Directory contents is returned if path points to directory.
  84. //
  85. // Use ServeFileUncompressed is you don't need serving compressed file contents.
  86. //
  87. // See also RequestCtx.SendFile.
  88. //
  89. // WARNING: do not pass any user supplied paths to this function!
  90. // WARNING: if path is based on user input users will be able to request
  91. // any file on your filesystem! Use fasthttp.FS with a sane Root instead.
  92. func ServeFile(ctx *RequestCtx, path string) {
  93. rootFSOnce.Do(func() {
  94. rootFSHandler = rootFS.NewRequestHandler()
  95. })
  96. if len(path) == 0 || !filepath.IsAbs(path) {
  97. // extend relative path to absolute path
  98. hasTrailingSlash := len(path) > 0 && (path[len(path)-1] == '/' || path[len(path)-1] == '\\')
  99. var err error
  100. path = filepath.FromSlash(path)
  101. if path, err = filepath.Abs(path); err != nil {
  102. ctx.Logger().Printf("cannot resolve path %q to absolute file path: %v", path, err)
  103. ctx.Error("Internal Server Error", StatusInternalServerError)
  104. return
  105. }
  106. if hasTrailingSlash {
  107. path += "/"
  108. }
  109. }
  110. // convert the path to forward slashes regardless the OS in order to set the URI properly
  111. // the handler will convert back to OS path separator before opening the file
  112. path = filepath.ToSlash(path)
  113. ctx.Request.SetRequestURI(path)
  114. rootFSHandler(ctx)
  115. }
  116. var (
  117. rootFSOnce sync.Once
  118. rootFS = &FS{
  119. Root: "",
  120. AllowEmptyRoot: true,
  121. GenerateIndexPages: true,
  122. Compress: true,
  123. CompressBrotli: true,
  124. AcceptByteRange: true,
  125. }
  126. rootFSHandler RequestHandler
  127. )
  128. // ServeFS returns HTTP response containing compressed file contents from the given fs.FS's path.
  129. //
  130. // HTTP response may contain uncompressed file contents in the following cases:
  131. //
  132. // - Missing 'Accept-Encoding: gzip' request header.
  133. // - No write access to directory containing the file.
  134. //
  135. // Directory contents is returned if path points to directory.
  136. //
  137. // See also ServeFile.
  138. func ServeFS(ctx *RequestCtx, filesystem fs.FS, path string) {
  139. f := &FS{
  140. FS: filesystem,
  141. Root: "",
  142. AllowEmptyRoot: true,
  143. GenerateIndexPages: true,
  144. Compress: true,
  145. CompressBrotli: true,
  146. AcceptByteRange: true,
  147. }
  148. handler := f.NewRequestHandler()
  149. ctx.Request.SetRequestURI(path)
  150. handler(ctx)
  151. }
  152. // PathRewriteFunc must return new request path based on arbitrary ctx
  153. // info such as ctx.Path().
  154. //
  155. // Path rewriter is used in FS for translating the current request
  156. // to the local filesystem path relative to FS.Root.
  157. //
  158. // The returned path must not contain '/../' substrings due to security reasons,
  159. // since such paths may refer files outside FS.Root.
  160. //
  161. // The returned path may refer to ctx members. For example, ctx.Path().
  162. type PathRewriteFunc func(ctx *RequestCtx) []byte
  163. // NewVHostPathRewriter returns path rewriter, which strips slashesCount
  164. // leading slashes from the path and prepends the path with request's host,
  165. // thus simplifying virtual hosting for static files.
  166. //
  167. // Examples:
  168. //
  169. // - host=foobar.com, slashesCount=0, original path="/foo/bar".
  170. // Resulting path: "/foobar.com/foo/bar"
  171. //
  172. // - host=img.aaa.com, slashesCount=1, original path="/images/123/456.jpg"
  173. // Resulting path: "/img.aaa.com/123/456.jpg"
  174. func NewVHostPathRewriter(slashesCount int) PathRewriteFunc {
  175. return func(ctx *RequestCtx) []byte {
  176. path := stripLeadingSlashes(ctx.Path(), slashesCount)
  177. host := ctx.Host()
  178. if n := bytes.IndexByte(host, '/'); n >= 0 {
  179. host = nil
  180. }
  181. if len(host) == 0 {
  182. host = strInvalidHost
  183. }
  184. b := bytebufferpool.Get()
  185. b.B = append(b.B, '/')
  186. b.B = append(b.B, host...)
  187. b.B = append(b.B, path...)
  188. ctx.URI().SetPathBytes(b.B)
  189. bytebufferpool.Put(b)
  190. return ctx.Path()
  191. }
  192. }
  193. var strInvalidHost = []byte("invalid-host")
  194. // NewPathSlashesStripper returns path rewriter, which strips slashesCount
  195. // leading slashes from the path.
  196. //
  197. // Examples:
  198. //
  199. // - slashesCount = 0, original path: "/foo/bar", result: "/foo/bar"
  200. // - slashesCount = 1, original path: "/foo/bar", result: "/bar"
  201. // - slashesCount = 2, original path: "/foo/bar", result: ""
  202. //
  203. // The returned path rewriter may be used as FS.PathRewrite .
  204. func NewPathSlashesStripper(slashesCount int) PathRewriteFunc {
  205. return func(ctx *RequestCtx) []byte {
  206. return stripLeadingSlashes(ctx.Path(), slashesCount)
  207. }
  208. }
  209. // NewPathPrefixStripper returns path rewriter, which removes prefixSize bytes
  210. // from the path prefix.
  211. //
  212. // Examples:
  213. //
  214. // - prefixSize = 0, original path: "/foo/bar", result: "/foo/bar"
  215. // - prefixSize = 3, original path: "/foo/bar", result: "o/bar"
  216. // - prefixSize = 7, original path: "/foo/bar", result: "r"
  217. //
  218. // The returned path rewriter may be used as FS.PathRewrite .
  219. func NewPathPrefixStripper(prefixSize int) PathRewriteFunc {
  220. return func(ctx *RequestCtx) []byte {
  221. path := ctx.Path()
  222. if len(path) >= prefixSize {
  223. path = path[prefixSize:]
  224. }
  225. return path
  226. }
  227. }
  228. // FS represents settings for request handler serving static files
  229. // from the local filesystem.
  230. //
  231. // It is prohibited copying FS values. Create new values instead.
  232. type FS struct {
  233. noCopy noCopy
  234. // FS is filesystem to serve files from. eg: embed.FS os.DirFS
  235. FS fs.FS
  236. // Path to the root directory to serve files from.
  237. Root string
  238. // AllowEmptyRoot controls what happens when Root is empty. When false (default) it will default to the
  239. // current working directory. An empty root is mostly useful when you want to use absolute paths
  240. // on windows that are on different filesystems. On linux setting your Root to "/" already allows you to use
  241. // absolute paths on any filesystem.
  242. AllowEmptyRoot bool
  243. // List of index file names to try opening during directory access.
  244. //
  245. // For example:
  246. //
  247. // * index.html
  248. // * index.htm
  249. // * my-super-index.xml
  250. //
  251. // By default the list is empty.
  252. IndexNames []string
  253. // Index pages for directories without files matching IndexNames
  254. // are automatically generated if set.
  255. //
  256. // Directory index generation may be quite slow for directories
  257. // with many files (more than 1K), so it is discouraged enabling
  258. // index pages' generation for such directories.
  259. //
  260. // By default index pages aren't generated.
  261. GenerateIndexPages bool
  262. // Transparently compresses responses if set to true.
  263. //
  264. // The server tries minimizing CPU usage by caching compressed files.
  265. // It adds CompressedFileSuffix suffix to the original file name and
  266. // tries saving the resulting compressed file under the new file name.
  267. // So it is advisable to give the server write access to Root
  268. // and to all inner folders in order to minimize CPU usage when serving
  269. // compressed responses.
  270. //
  271. // Transparent compression is disabled by default.
  272. Compress bool
  273. // Uses brotli encoding and fallbacks to gzip in responses if set to true, uses gzip if set to false.
  274. //
  275. // This value has sense only if Compress is set.
  276. //
  277. // Brotli encoding is disabled by default.
  278. CompressBrotli bool
  279. // Path to the compressed root directory to serve files from. If this value
  280. // is empty, Root is used.
  281. CompressRoot string
  282. // Enables byte range requests if set to true.
  283. //
  284. // Byte range requests are disabled by default.
  285. AcceptByteRange bool
  286. // Path rewriting function.
  287. //
  288. // By default request path is not modified.
  289. PathRewrite PathRewriteFunc
  290. // PathNotFound fires when file is not found in filesystem
  291. // this functions tries to replace "Cannot open requested path"
  292. // server response giving to the programmer the control of server flow.
  293. //
  294. // By default PathNotFound returns
  295. // "Cannot open requested path"
  296. PathNotFound RequestHandler
  297. // SkipCache if true, will cache no file handler.
  298. //
  299. // By default is false.
  300. SkipCache bool
  301. // Expiration duration for inactive file handlers.
  302. //
  303. // FSHandlerCacheDuration is used by default.
  304. CacheDuration time.Duration
  305. // Suffix to add to the name of cached compressed file.
  306. //
  307. // This value has sense only if Compress is set.
  308. //
  309. // FSCompressedFileSuffix is used by default.
  310. CompressedFileSuffix string
  311. // Suffixes list to add to compressedFileSuffix depending on encoding
  312. //
  313. // This value has sense only if Compress is set.
  314. //
  315. // FSCompressedFileSuffixes is used by default.
  316. CompressedFileSuffixes map[string]string
  317. // If CleanStop is set, the channel can be closed to stop the cleanup handlers
  318. // for the FS RequestHandlers created with NewRequestHandler.
  319. // NEVER close this channel while the handler is still being used!
  320. CleanStop chan struct{}
  321. once sync.Once
  322. h RequestHandler
  323. }
  324. // FSCompressedFileSuffix is the suffix FS adds to the original file names
  325. // when trying to store compressed file under the new file name.
  326. // See FS.Compress for details.
  327. const FSCompressedFileSuffix = ".fasthttp.gz"
  328. // FSCompressedFileSuffixes is the suffixes FS adds to the original file names depending on encoding
  329. // when trying to store compressed file under the new file name.
  330. // See FS.Compress for details.
  331. var FSCompressedFileSuffixes = map[string]string{
  332. "gzip": ".fasthttp.gz",
  333. "br": ".fasthttp.br",
  334. }
  335. // FSHandlerCacheDuration is the default expiration duration for inactive
  336. // file handlers opened by FS.
  337. const FSHandlerCacheDuration = 10 * time.Second
  338. // FSHandler returns request handler serving static files from
  339. // the given root folder.
  340. //
  341. // stripSlashes indicates how many leading slashes must be stripped
  342. // from requested path before searching requested file in the root folder.
  343. // Examples:
  344. //
  345. // - stripSlashes = 0, original path: "/foo/bar", result: "/foo/bar"
  346. // - stripSlashes = 1, original path: "/foo/bar", result: "/bar"
  347. // - stripSlashes = 2, original path: "/foo/bar", result: ""
  348. //
  349. // The returned request handler automatically generates index pages
  350. // for directories without index.html.
  351. //
  352. // The returned handler caches requested file handles
  353. // for FSHandlerCacheDuration.
  354. // Make sure your program has enough 'max open files' limit aka
  355. // 'ulimit -n' if root folder contains many files.
  356. //
  357. // Do not create multiple request handler instances for the same
  358. // (root, stripSlashes) arguments - just reuse a single instance.
  359. // Otherwise goroutine leak will occur.
  360. func FSHandler(root string, stripSlashes int) RequestHandler {
  361. fs := &FS{
  362. Root: root,
  363. IndexNames: []string{"index.html"},
  364. GenerateIndexPages: true,
  365. AcceptByteRange: true,
  366. }
  367. if stripSlashes > 0 {
  368. fs.PathRewrite = NewPathSlashesStripper(stripSlashes)
  369. }
  370. return fs.NewRequestHandler()
  371. }
  372. // NewRequestHandler returns new request handler with the given FS settings.
  373. //
  374. // The returned handler caches requested file handles
  375. // for FS.CacheDuration.
  376. // Make sure your program has enough 'max open files' limit aka
  377. // 'ulimit -n' if FS.Root folder contains many files.
  378. //
  379. // Do not create multiple request handlers from a single FS instance -
  380. // just reuse a single request handler.
  381. func (fs *FS) NewRequestHandler() RequestHandler {
  382. fs.once.Do(fs.initRequestHandler)
  383. return fs.h
  384. }
  385. func (fs *FS) normalizeRoot(root string) string {
  386. // fs.FS uses relative paths, that paths are slash-separated on all systems, even Windows.
  387. if fs.FS == nil {
  388. // Serve files from the current working directory if Root is empty or if Root is a relative path.
  389. if (!fs.AllowEmptyRoot && len(root) == 0) || (len(root) > 0 && !filepath.IsAbs(root)) {
  390. path, err := os.Getwd()
  391. if err != nil {
  392. path = "."
  393. }
  394. root = path + "/" + root
  395. }
  396. // convert the root directory slashes to the native format
  397. root = filepath.FromSlash(root)
  398. }
  399. // strip trailing slashes from the root path
  400. for len(root) > 0 && root[len(root)-1] == os.PathSeparator {
  401. root = root[:len(root)-1]
  402. }
  403. return root
  404. }
  405. func (fs *FS) initRequestHandler() {
  406. root := fs.normalizeRoot(fs.Root)
  407. compressRoot := fs.CompressRoot
  408. if len(compressRoot) == 0 {
  409. compressRoot = root
  410. } else {
  411. compressRoot = fs.normalizeRoot(compressRoot)
  412. }
  413. compressedFileSuffixes := fs.CompressedFileSuffixes
  414. if len(compressedFileSuffixes["br"]) == 0 || len(compressedFileSuffixes["gzip"]) == 0 ||
  415. compressedFileSuffixes["br"] == compressedFileSuffixes["gzip"] {
  416. // Copy global map
  417. compressedFileSuffixes = make(map[string]string, len(FSCompressedFileSuffixes))
  418. for k, v := range FSCompressedFileSuffixes {
  419. compressedFileSuffixes[k] = v
  420. }
  421. }
  422. if len(fs.CompressedFileSuffix) > 0 {
  423. compressedFileSuffixes["gzip"] = fs.CompressedFileSuffix
  424. compressedFileSuffixes["br"] = FSCompressedFileSuffixes["br"]
  425. }
  426. h := &fsHandler{
  427. filesystem: fs.FS,
  428. root: root,
  429. indexNames: fs.IndexNames,
  430. pathRewrite: fs.PathRewrite,
  431. generateIndexPages: fs.GenerateIndexPages,
  432. compress: fs.Compress,
  433. compressBrotli: fs.CompressBrotli,
  434. compressRoot: compressRoot,
  435. pathNotFound: fs.PathNotFound,
  436. acceptByteRange: fs.AcceptByteRange,
  437. compressedFileSuffixes: compressedFileSuffixes,
  438. }
  439. h.cacheManager = newCacheManager(fs)
  440. if h.filesystem == nil {
  441. h.filesystem = &osFS{} // It provides os.Open and os.Stat
  442. }
  443. fs.h = h.handleRequest
  444. }
  445. type fsHandler struct {
  446. filesystem fs.FS
  447. root string
  448. indexNames []string
  449. pathRewrite PathRewriteFunc
  450. pathNotFound RequestHandler
  451. generateIndexPages bool
  452. compress bool
  453. compressBrotli bool
  454. compressRoot string
  455. acceptByteRange bool
  456. compressedFileSuffixes map[string]string
  457. cacheManager cacheManager
  458. smallFileReaderPool sync.Pool
  459. }
  460. type fsFile struct {
  461. h *fsHandler
  462. f fs.File
  463. filename string // fs.FileInfo.Name() return filename, isn't filepath.
  464. dirIndex []byte
  465. contentType string
  466. contentLength int
  467. compressed bool
  468. lastModified time.Time
  469. lastModifiedStr []byte
  470. t time.Time
  471. readersCount int
  472. bigFiles []*bigFileReader
  473. bigFilesLock sync.Mutex
  474. }
  475. func (ff *fsFile) NewReader() (io.Reader, error) {
  476. if ff.isBig() {
  477. r, err := ff.bigFileReader()
  478. if err != nil {
  479. ff.decReadersCount()
  480. }
  481. return r, err
  482. }
  483. return ff.smallFileReader()
  484. }
  485. func (ff *fsFile) smallFileReader() (io.Reader, error) {
  486. v := ff.h.smallFileReaderPool.Get()
  487. if v == nil {
  488. v = &fsSmallFileReader{}
  489. }
  490. r := v.(*fsSmallFileReader)
  491. r.ff = ff
  492. r.endPos = ff.contentLength
  493. if r.startPos > 0 {
  494. return nil, errors.New("bug: fsSmallFileReader with non-nil startPos found in the pool")
  495. }
  496. return r, nil
  497. }
  498. // files bigger than this size are sent with sendfile
  499. const maxSmallFileSize = 2 * 4096
  500. func (ff *fsFile) isBig() bool {
  501. if _, ok := ff.h.filesystem.(*osFS); !ok { // fs.FS only uses bigFileReader, memory cache uses fsSmallFileReader
  502. return ff.f != nil
  503. }
  504. return ff.contentLength > maxSmallFileSize && len(ff.dirIndex) == 0
  505. }
  506. func (ff *fsFile) bigFileReader() (io.Reader, error) {
  507. if ff.f == nil {
  508. return nil, errors.New("bug: ff.f must be non-nil in bigFileReader")
  509. }
  510. var r io.Reader
  511. ff.bigFilesLock.Lock()
  512. n := len(ff.bigFiles)
  513. if n > 0 {
  514. r = ff.bigFiles[n-1]
  515. ff.bigFiles = ff.bigFiles[:n-1]
  516. }
  517. ff.bigFilesLock.Unlock()
  518. if r != nil {
  519. return r, nil
  520. }
  521. f, err := ff.h.filesystem.Open(ff.filename)
  522. if err != nil {
  523. return nil, fmt.Errorf("cannot open already opened file: %w", err)
  524. }
  525. return &bigFileReader{
  526. f: f,
  527. ff: ff,
  528. r: f,
  529. }, nil
  530. }
  531. func (ff *fsFile) Release() {
  532. if ff.f != nil {
  533. _ = ff.f.Close()
  534. if ff.isBig() {
  535. ff.bigFilesLock.Lock()
  536. for _, r := range ff.bigFiles {
  537. _ = r.f.Close()
  538. }
  539. ff.bigFilesLock.Unlock()
  540. }
  541. }
  542. }
  543. func (ff *fsFile) decReadersCount() {
  544. ff.h.cacheManager.WithLock(func() {
  545. ff.readersCount--
  546. if ff.readersCount < 0 {
  547. ff.readersCount = 0
  548. }
  549. })
  550. }
  551. // bigFileReader attempts to trigger sendfile
  552. // for sending big files over the wire.
  553. type bigFileReader struct {
  554. f fs.File
  555. ff *fsFile
  556. r io.Reader
  557. lr io.LimitedReader
  558. }
  559. func (r *bigFileReader) UpdateByteRange(startPos, endPos int) error {
  560. seeker, ok := r.f.(io.Seeker)
  561. if !ok {
  562. return errors.New("must implement io.Seeker")
  563. }
  564. if _, err := seeker.Seek(int64(startPos), io.SeekStart); err != nil {
  565. return err
  566. }
  567. r.r = &r.lr
  568. r.lr.R = r.f
  569. r.lr.N = int64(endPos - startPos + 1)
  570. return nil
  571. }
  572. func (r *bigFileReader) Read(p []byte) (int, error) {
  573. return r.r.Read(p)
  574. }
  575. func (r *bigFileReader) WriteTo(w io.Writer) (int64, error) {
  576. if rf, ok := w.(io.ReaderFrom); ok {
  577. // fast path. Send file must be triggered
  578. return rf.ReadFrom(r.r)
  579. }
  580. // slow path
  581. return copyZeroAlloc(w, r.r)
  582. }
  583. func (r *bigFileReader) Close() error {
  584. r.r = r.f
  585. seeker, ok := r.f.(io.Seeker)
  586. if !ok {
  587. _ = r.f.Close()
  588. return errors.New("must implement io.Seeker")
  589. }
  590. n, err := seeker.Seek(0, io.SeekStart)
  591. if err == nil {
  592. if n == 0 {
  593. ff := r.ff
  594. ff.bigFilesLock.Lock()
  595. ff.bigFiles = append(ff.bigFiles, r)
  596. ff.bigFilesLock.Unlock()
  597. } else {
  598. _ = r.f.Close()
  599. err = errors.New("bug: File.Seek(0, io.SeekStart) returned (non-zero, nil)")
  600. }
  601. } else {
  602. _ = r.f.Close()
  603. }
  604. r.ff.decReadersCount()
  605. return err
  606. }
  607. type fsSmallFileReader struct {
  608. ff *fsFile
  609. startPos int
  610. endPos int
  611. }
  612. func (r *fsSmallFileReader) Close() error {
  613. ff := r.ff
  614. ff.decReadersCount()
  615. r.ff = nil
  616. r.startPos = 0
  617. r.endPos = 0
  618. ff.h.smallFileReaderPool.Put(r)
  619. return nil
  620. }
  621. func (r *fsSmallFileReader) UpdateByteRange(startPos, endPos int) error {
  622. r.startPos = startPos
  623. r.endPos = endPos + 1
  624. return nil
  625. }
  626. func (r *fsSmallFileReader) Read(p []byte) (int, error) {
  627. tailLen := r.endPos - r.startPos
  628. if tailLen <= 0 {
  629. return 0, io.EOF
  630. }
  631. if len(p) > tailLen {
  632. p = p[:tailLen]
  633. }
  634. ff := r.ff
  635. if ff.f != nil {
  636. ra, ok := ff.f.(io.ReaderAt)
  637. if !ok {
  638. return 0, errors.New("must implement io.ReaderAt")
  639. }
  640. n, err := ra.ReadAt(p, int64(r.startPos))
  641. r.startPos += n
  642. return n, err
  643. }
  644. n := copy(p, ff.dirIndex[r.startPos:])
  645. r.startPos += n
  646. return n, nil
  647. }
  648. func (r *fsSmallFileReader) WriteTo(w io.Writer) (int64, error) {
  649. ff := r.ff
  650. var n int
  651. var err error
  652. if ff.f == nil {
  653. n, err = w.Write(ff.dirIndex[r.startPos:r.endPos])
  654. return int64(n), err
  655. }
  656. if rf, ok := w.(io.ReaderFrom); ok {
  657. return rf.ReadFrom(r)
  658. }
  659. curPos := r.startPos
  660. bufv := copyBufPool.Get()
  661. buf := bufv.([]byte)
  662. for err == nil {
  663. tailLen := r.endPos - curPos
  664. if tailLen <= 0 {
  665. break
  666. }
  667. if len(buf) > tailLen {
  668. buf = buf[:tailLen]
  669. }
  670. ra, ok := ff.f.(io.ReaderAt)
  671. if !ok {
  672. return 0, errors.New("must implement io.ReaderAt")
  673. }
  674. n, err = ra.ReadAt(buf, int64(curPos))
  675. nw, errw := w.Write(buf[:n])
  676. curPos += nw
  677. if errw == nil && nw != n {
  678. errw = errors.New("bug: Write(p) returned (n, nil), where n != len(p)")
  679. }
  680. if err == nil {
  681. err = errw
  682. }
  683. }
  684. copyBufPool.Put(bufv)
  685. if err == io.EOF {
  686. err = nil
  687. }
  688. return int64(curPos - r.startPos), err
  689. }
  690. type cacheManager interface {
  691. WithLock(work func())
  692. GetFileFromCache(cacheKind CacheKind, path string) (*fsFile, bool)
  693. SetFileToCache(cacheKind CacheKind, path string, ff *fsFile) *fsFile
  694. }
  695. var (
  696. _ cacheManager = (*inMemoryCacheManager)(nil)
  697. _ cacheManager = (*noopCacheManager)(nil)
  698. )
  699. type CacheKind uint8
  700. const (
  701. defaultCacheKind CacheKind = iota
  702. brotliCacheKind
  703. gzipCacheKind
  704. )
  705. func newCacheManager(fs *FS) cacheManager {
  706. if fs.SkipCache {
  707. return &noopCacheManager{}
  708. }
  709. cacheDuration := fs.CacheDuration
  710. if cacheDuration <= 0 {
  711. cacheDuration = FSHandlerCacheDuration
  712. }
  713. instance := &inMemoryCacheManager{
  714. cacheDuration: cacheDuration,
  715. cache: make(map[string]*fsFile),
  716. cacheBrotli: make(map[string]*fsFile),
  717. cacheGzip: make(map[string]*fsFile),
  718. }
  719. go instance.handleCleanCache(fs.CleanStop)
  720. return instance
  721. }
  722. type noopCacheManager struct {
  723. cacheLock sync.Mutex
  724. }
  725. func (n *noopCacheManager) WithLock(work func()) {
  726. n.cacheLock.Lock()
  727. work()
  728. n.cacheLock.Unlock()
  729. }
  730. func (*noopCacheManager) GetFileFromCache(cacheKind CacheKind, path string) (*fsFile, bool) {
  731. return nil, false
  732. }
  733. func (*noopCacheManager) SetFileToCache(cacheKind CacheKind, path string, ff *fsFile) *fsFile {
  734. return ff
  735. }
  736. type inMemoryCacheManager struct {
  737. cacheDuration time.Duration
  738. cache map[string]*fsFile
  739. cacheBrotli map[string]*fsFile
  740. cacheGzip map[string]*fsFile
  741. cacheLock sync.Mutex
  742. }
  743. func (cm *inMemoryCacheManager) WithLock(work func()) {
  744. cm.cacheLock.Lock()
  745. work()
  746. cm.cacheLock.Unlock()
  747. }
  748. func (cm *inMemoryCacheManager) getFsCache(cacheKind CacheKind) map[string]*fsFile {
  749. fileCache := cm.cache
  750. switch cacheKind {
  751. case brotliCacheKind:
  752. fileCache = cm.cacheBrotli
  753. case gzipCacheKind:
  754. fileCache = cm.cacheGzip
  755. }
  756. return fileCache
  757. }
  758. func (cm *inMemoryCacheManager) GetFileFromCache(cacheKind CacheKind, path string) (*fsFile, bool) {
  759. fileCache := cm.getFsCache(cacheKind)
  760. cm.cacheLock.Lock()
  761. ff, ok := fileCache[path]
  762. if ok {
  763. ff.readersCount++
  764. }
  765. cm.cacheLock.Unlock()
  766. return ff, ok
  767. }
  768. func (cm *inMemoryCacheManager) SetFileToCache(cacheKind CacheKind, path string, ff *fsFile) *fsFile {
  769. fileCache := cm.getFsCache(cacheKind)
  770. cm.cacheLock.Lock()
  771. ff1, ok := fileCache[path]
  772. if !ok {
  773. fileCache[path] = ff
  774. ff.readersCount++
  775. } else {
  776. ff1.readersCount++
  777. }
  778. cm.cacheLock.Unlock()
  779. if ok {
  780. // The file has been already opened by another
  781. // goroutine, so close the current file and use
  782. // the file opened by another goroutine instead.
  783. ff.Release()
  784. ff = ff1
  785. }
  786. return ff
  787. }
  788. func (cm *inMemoryCacheManager) handleCleanCache(cleanStop chan struct{}) {
  789. var pendingFiles []*fsFile
  790. clean := func() {
  791. pendingFiles = cm.cleanCache(pendingFiles)
  792. }
  793. if cleanStop != nil {
  794. t := time.NewTicker(cm.cacheDuration / 2)
  795. for {
  796. select {
  797. case <-t.C:
  798. clean()
  799. case _, stillOpen := <-cleanStop:
  800. // Ignore values send on the channel, only stop when it is closed.
  801. if !stillOpen {
  802. t.Stop()
  803. return
  804. }
  805. }
  806. }
  807. }
  808. for {
  809. time.Sleep(cm.cacheDuration / 2)
  810. clean()
  811. }
  812. }
  813. func (cm *inMemoryCacheManager) cleanCache(pendingFiles []*fsFile) []*fsFile {
  814. var filesToRelease []*fsFile
  815. cm.cacheLock.Lock()
  816. // Close files which couldn't be closed before due to non-zero
  817. // readers count on the previous run.
  818. var remainingFiles []*fsFile
  819. for _, ff := range pendingFiles {
  820. if ff.readersCount > 0 {
  821. remainingFiles = append(remainingFiles, ff)
  822. } else {
  823. filesToRelease = append(filesToRelease, ff)
  824. }
  825. }
  826. pendingFiles = remainingFiles
  827. pendingFiles, filesToRelease = cleanCacheNolock(cm.cache, pendingFiles, filesToRelease, cm.cacheDuration)
  828. pendingFiles, filesToRelease = cleanCacheNolock(cm.cacheBrotli, pendingFiles, filesToRelease, cm.cacheDuration)
  829. pendingFiles, filesToRelease = cleanCacheNolock(cm.cacheGzip, pendingFiles, filesToRelease, cm.cacheDuration)
  830. cm.cacheLock.Unlock()
  831. for _, ff := range filesToRelease {
  832. ff.Release()
  833. }
  834. return pendingFiles
  835. }
  836. func cleanCacheNolock(cache map[string]*fsFile, pendingFiles, filesToRelease []*fsFile, cacheDuration time.Duration) ([]*fsFile, []*fsFile) {
  837. t := time.Now()
  838. for k, ff := range cache {
  839. if t.Sub(ff.t) > cacheDuration {
  840. if ff.readersCount > 0 {
  841. // There are pending readers on stale file handle,
  842. // so we cannot close it. Put it into pendingFiles
  843. // so it will be closed later.
  844. pendingFiles = append(pendingFiles, ff)
  845. } else {
  846. filesToRelease = append(filesToRelease, ff)
  847. }
  848. delete(cache, k)
  849. }
  850. }
  851. return pendingFiles, filesToRelease
  852. }
  853. func (h *fsHandler) pathToFilePath(path string) string {
  854. if _, ok := h.filesystem.(*osFS); !ok {
  855. if len(path) < 1 {
  856. return path
  857. }
  858. return path[1:]
  859. }
  860. return filepath.FromSlash(h.root + path)
  861. }
  862. func (h *fsHandler) filePathToCompressed(filePath string) string {
  863. if h.root == h.compressRoot {
  864. return filePath
  865. }
  866. if !strings.HasPrefix(filePath, h.root) {
  867. return filePath
  868. }
  869. return filepath.FromSlash(h.compressRoot + filePath[len(h.root):])
  870. }
  871. func (h *fsHandler) handleRequest(ctx *RequestCtx) {
  872. var path []byte
  873. if h.pathRewrite != nil {
  874. path = h.pathRewrite(ctx)
  875. } else {
  876. path = ctx.Path()
  877. }
  878. hasTrailingSlash := len(path) > 0 && path[len(path)-1] == '/'
  879. path = stripTrailingSlashes(path)
  880. if n := bytes.IndexByte(path, 0); n >= 0 {
  881. ctx.Logger().Printf("cannot serve path with nil byte at position %d: %q", n, path)
  882. ctx.Error("Are you a hacker?", StatusBadRequest)
  883. return
  884. }
  885. if h.pathRewrite != nil {
  886. // There is no need to check for '/../' if path = ctx.Path(),
  887. // since ctx.Path must normalize and sanitize the path.
  888. if n := bytes.Index(path, strSlashDotDotSlash); n >= 0 {
  889. ctx.Logger().Printf("cannot serve path with '/../' at position %d due to security reasons: %q", n, path)
  890. ctx.Error("Internal Server Error", StatusInternalServerError)
  891. return
  892. }
  893. }
  894. mustCompress := false
  895. fileCacheKind := defaultCacheKind
  896. fileEncoding := ""
  897. byteRange := ctx.Request.Header.peek(strRange)
  898. if len(byteRange) == 0 && h.compress {
  899. if h.compressBrotli && ctx.Request.Header.HasAcceptEncodingBytes(strBr) {
  900. mustCompress = true
  901. fileCacheKind = brotliCacheKind
  902. fileEncoding = "br"
  903. } else if ctx.Request.Header.HasAcceptEncodingBytes(strGzip) {
  904. mustCompress = true
  905. fileCacheKind = gzipCacheKind
  906. fileEncoding = "gzip"
  907. }
  908. }
  909. pathStr := string(path)
  910. ff, ok := h.cacheManager.GetFileFromCache(fileCacheKind, pathStr)
  911. if !ok {
  912. filePath := h.pathToFilePath(pathStr)
  913. var err error
  914. ff, err = h.openFSFile(filePath, mustCompress, fileEncoding)
  915. if mustCompress && err == errNoCreatePermission {
  916. ctx.Logger().Printf("insufficient permissions for saving compressed file for %q. Serving uncompressed file. "+
  917. "Allow write access to the directory with this file in order to improve fasthttp performance", filePath)
  918. mustCompress = false
  919. ff, err = h.openFSFile(filePath, mustCompress, fileEncoding)
  920. }
  921. if err == errDirIndexRequired {
  922. if !hasTrailingSlash {
  923. ctx.RedirectBytes(append(path, '/'), StatusFound)
  924. return
  925. }
  926. ff, err = h.openIndexFile(ctx, filePath, mustCompress, fileEncoding)
  927. if err != nil {
  928. ctx.Logger().Printf("cannot open dir index %q: %v", filePath, err)
  929. ctx.Error("Directory index is forbidden", StatusForbidden)
  930. return
  931. }
  932. } else if err != nil {
  933. ctx.Logger().Printf("cannot open file %q: %v", filePath, err)
  934. if h.pathNotFound == nil {
  935. ctx.Error("Cannot open requested path", StatusNotFound)
  936. } else {
  937. ctx.SetStatusCode(StatusNotFound)
  938. h.pathNotFound(ctx)
  939. }
  940. return
  941. }
  942. ff = h.cacheManager.SetFileToCache(fileCacheKind, pathStr, ff)
  943. }
  944. if !ctx.IfModifiedSince(ff.lastModified) {
  945. ff.decReadersCount()
  946. ctx.NotModified()
  947. return
  948. }
  949. r, err := ff.NewReader()
  950. if err != nil {
  951. ctx.Logger().Printf("cannot obtain file reader for path=%q: %v", path, err)
  952. ctx.Error("Internal Server Error", StatusInternalServerError)
  953. return
  954. }
  955. hdr := &ctx.Response.Header
  956. if ff.compressed {
  957. if fileEncoding == "br" {
  958. hdr.SetContentEncodingBytes(strBr)
  959. } else if fileEncoding == "gzip" {
  960. hdr.SetContentEncodingBytes(strGzip)
  961. }
  962. }
  963. statusCode := StatusOK
  964. contentLength := ff.contentLength
  965. if h.acceptByteRange {
  966. hdr.setNonSpecial(strAcceptRanges, strBytes)
  967. if len(byteRange) > 0 {
  968. startPos, endPos, err := ParseByteRange(byteRange, contentLength)
  969. if err != nil {
  970. _ = r.(io.Closer).Close()
  971. ctx.Logger().Printf("cannot parse byte range %q for path=%q: %v", byteRange, path, err)
  972. ctx.Error("Range Not Satisfiable", StatusRequestedRangeNotSatisfiable)
  973. return
  974. }
  975. if err = r.(byteRangeUpdater).UpdateByteRange(startPos, endPos); err != nil {
  976. _ = r.(io.Closer).Close()
  977. ctx.Logger().Printf("cannot seek byte range %q for path=%q: %v", byteRange, path, err)
  978. ctx.Error("Internal Server Error", StatusInternalServerError)
  979. return
  980. }
  981. hdr.SetContentRange(startPos, endPos, contentLength)
  982. contentLength = endPos - startPos + 1
  983. statusCode = StatusPartialContent
  984. }
  985. }
  986. hdr.setNonSpecial(strLastModified, ff.lastModifiedStr)
  987. if !ctx.IsHead() {
  988. ctx.SetBodyStream(r, contentLength)
  989. } else {
  990. ctx.Response.ResetBody()
  991. ctx.Response.SkipBody = true
  992. ctx.Response.Header.SetContentLength(contentLength)
  993. if rc, ok := r.(io.Closer); ok {
  994. if err := rc.Close(); err != nil {
  995. ctx.Logger().Printf("cannot close file reader: %v", err)
  996. ctx.Error("Internal Server Error", StatusInternalServerError)
  997. return
  998. }
  999. }
  1000. }
  1001. hdr.noDefaultContentType = true
  1002. if len(hdr.ContentType()) == 0 {
  1003. ctx.SetContentType(ff.contentType)
  1004. }
  1005. ctx.SetStatusCode(statusCode)
  1006. }
  1007. type byteRangeUpdater interface {
  1008. UpdateByteRange(startPos, endPos int) error
  1009. }
  1010. // ParseByteRange parses 'Range: bytes=...' header value.
  1011. //
  1012. // It follows https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 .
  1013. func ParseByteRange(byteRange []byte, contentLength int) (startPos, endPos int, err error) {
  1014. b := byteRange
  1015. if !bytes.HasPrefix(b, strBytes) {
  1016. return 0, 0, fmt.Errorf("unsupported range units: %q. Expecting %q", byteRange, strBytes)
  1017. }
  1018. b = b[len(strBytes):]
  1019. if len(b) == 0 || b[0] != '=' {
  1020. return 0, 0, fmt.Errorf("missing byte range in %q", byteRange)
  1021. }
  1022. b = b[1:]
  1023. n := bytes.IndexByte(b, '-')
  1024. if n < 0 {
  1025. return 0, 0, fmt.Errorf("missing the end position of byte range in %q", byteRange)
  1026. }
  1027. if n == 0 {
  1028. v, err := ParseUint(b[n+1:])
  1029. if err != nil {
  1030. return 0, 0, err
  1031. }
  1032. startPos := contentLength - v
  1033. if startPos < 0 {
  1034. startPos = 0
  1035. }
  1036. return startPos, contentLength - 1, nil
  1037. }
  1038. if startPos, err = ParseUint(b[:n]); err != nil {
  1039. return 0, 0, err
  1040. }
  1041. if startPos >= contentLength {
  1042. return 0, 0, fmt.Errorf("the start position of byte range cannot exceed %d. byte range %q", contentLength-1, byteRange)
  1043. }
  1044. b = b[n+1:]
  1045. if len(b) == 0 {
  1046. return startPos, contentLength - 1, nil
  1047. }
  1048. if endPos, err = ParseUint(b); err != nil {
  1049. return 0, 0, err
  1050. }
  1051. if endPos >= contentLength {
  1052. endPos = contentLength - 1
  1053. }
  1054. if endPos < startPos {
  1055. return 0, 0, fmt.Errorf("the start position of byte range cannot exceed the end position. byte range %q", byteRange)
  1056. }
  1057. return startPos, endPos, nil
  1058. }
  1059. func (h *fsHandler) openIndexFile(ctx *RequestCtx, dirPath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
  1060. for _, indexName := range h.indexNames {
  1061. indexFilePath := dirPath + "/" + indexName
  1062. ff, err := h.openFSFile(indexFilePath, mustCompress, fileEncoding)
  1063. if err == nil {
  1064. return ff, nil
  1065. }
  1066. if !errors.Is(err, fs.ErrNotExist) {
  1067. return nil, fmt.Errorf("cannot open file %q: %w", indexFilePath, err)
  1068. }
  1069. }
  1070. if !h.generateIndexPages {
  1071. return nil, fmt.Errorf("cannot access directory without index page. Directory %q", dirPath)
  1072. }
  1073. return h.createDirIndex(ctx, dirPath, mustCompress, fileEncoding)
  1074. }
  1075. var (
  1076. errDirIndexRequired = errors.New("directory index required")
  1077. errNoCreatePermission = errors.New("no 'create file' permissions")
  1078. )
  1079. func (h *fsHandler) createDirIndex(ctx *RequestCtx, dirPath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
  1080. w := &bytebufferpool.ByteBuffer{}
  1081. base := ctx.URI()
  1082. basePathEscaped := html.EscapeString(string(base.Path()))
  1083. _, _ = fmt.Fprintf(w, "<html><head><title>%s</title><style>.dir { font-weight: bold }</style></head><body>", basePathEscaped)
  1084. _, _ = fmt.Fprintf(w, "<h1>%s</h1>", basePathEscaped)
  1085. _, _ = fmt.Fprintf(w, "<ul>")
  1086. if len(basePathEscaped) > 1 {
  1087. var parentURI URI
  1088. base.CopyTo(&parentURI)
  1089. parentURI.Update(string(base.Path()) + "/..")
  1090. parentPathEscaped := html.EscapeString(string(parentURI.Path()))
  1091. _, _ = fmt.Fprintf(w, `<li><a href="%s" class="dir">..</a></li>`, parentPathEscaped)
  1092. }
  1093. dirEntries, err := fs.ReadDir(h.filesystem, dirPath)
  1094. if err != nil {
  1095. return nil, err
  1096. }
  1097. fm := make(map[string]fs.FileInfo, len(dirEntries))
  1098. filenames := make([]string, 0, len(dirEntries))
  1099. nestedContinue:
  1100. for _, de := range dirEntries {
  1101. name := de.Name()
  1102. for _, cfs := range h.compressedFileSuffixes {
  1103. if strings.HasSuffix(name, cfs) {
  1104. // Do not show compressed files on index page.
  1105. continue nestedContinue
  1106. }
  1107. }
  1108. fi, err := de.Info()
  1109. if err != nil {
  1110. ctx.Logger().Printf("cannot fetch information from dir entry %q: %v, skip", name, err)
  1111. continue nestedContinue
  1112. }
  1113. fm[name] = fi
  1114. filenames = append(filenames, name)
  1115. }
  1116. var u URI
  1117. base.CopyTo(&u)
  1118. u.Update(string(u.Path()) + "/")
  1119. sort.Strings(filenames)
  1120. for _, name := range filenames {
  1121. u.Update(name)
  1122. pathEscaped := html.EscapeString(string(u.Path()))
  1123. fi := fm[name]
  1124. auxStr := "dir"
  1125. className := "dir"
  1126. if !fi.IsDir() {
  1127. auxStr = fmt.Sprintf("file, %d bytes", fi.Size())
  1128. className = "file"
  1129. }
  1130. _, _ = fmt.Fprintf(w, `<li><a href="%s" class="%s">%s</a>, %s, last modified %s</li>`,
  1131. pathEscaped, className, html.EscapeString(name), auxStr, fsModTime(fi.ModTime()))
  1132. }
  1133. _, _ = fmt.Fprintf(w, "</ul></body></html>")
  1134. if mustCompress {
  1135. var zbuf bytebufferpool.ByteBuffer
  1136. if fileEncoding == "br" {
  1137. zbuf.B = AppendBrotliBytesLevel(zbuf.B, w.B, CompressDefaultCompression)
  1138. } else if fileEncoding == "gzip" {
  1139. zbuf.B = AppendGzipBytesLevel(zbuf.B, w.B, CompressDefaultCompression)
  1140. }
  1141. w = &zbuf
  1142. }
  1143. dirIndex := w.B
  1144. lastModified := time.Now()
  1145. ff := &fsFile{
  1146. h: h,
  1147. dirIndex: dirIndex,
  1148. contentType: "text/html; charset=utf-8",
  1149. contentLength: len(dirIndex),
  1150. compressed: mustCompress,
  1151. lastModified: lastModified,
  1152. lastModifiedStr: AppendHTTPDate(nil, lastModified),
  1153. t: lastModified,
  1154. }
  1155. return ff, nil
  1156. }
  1157. const (
  1158. fsMinCompressRatio = 0.8
  1159. fsMaxCompressibleFileSize = 8 * 1024 * 1024
  1160. )
  1161. func (h *fsHandler) compressAndOpenFSFile(filePath string, fileEncoding string) (*fsFile, error) {
  1162. f, err := h.filesystem.Open(filePath)
  1163. if err != nil {
  1164. return nil, err
  1165. }
  1166. fileInfo, err := f.Stat()
  1167. if err != nil {
  1168. _ = f.Close()
  1169. return nil, fmt.Errorf("cannot obtain info for file %q: %w", filePath, err)
  1170. }
  1171. if fileInfo.IsDir() {
  1172. _ = f.Close()
  1173. return nil, errDirIndexRequired
  1174. }
  1175. if strings.HasSuffix(filePath, h.compressedFileSuffixes[fileEncoding]) ||
  1176. fileInfo.Size() > fsMaxCompressibleFileSize ||
  1177. !isFileCompressible(f, fsMinCompressRatio) {
  1178. return h.newFSFile(f, fileInfo, false, filePath, "")
  1179. }
  1180. compressedFilePath := h.filePathToCompressed(filePath)
  1181. if _, ok := h.filesystem.(*osFS); !ok {
  1182. return h.newCompressedFSFileCache(f, fileInfo, compressedFilePath, fileEncoding)
  1183. }
  1184. if compressedFilePath != filePath {
  1185. if err := os.MkdirAll(filepath.Dir(compressedFilePath), os.ModePerm); err != nil {
  1186. return nil, err
  1187. }
  1188. }
  1189. compressedFilePath += h.compressedFileSuffixes[fileEncoding]
  1190. absPath, err := filepath.Abs(compressedFilePath)
  1191. if err != nil {
  1192. _ = f.Close()
  1193. return nil, fmt.Errorf("cannot determine absolute path for %q: %v", compressedFilePath, err)
  1194. }
  1195. flock := getFileLock(absPath)
  1196. flock.Lock()
  1197. ff, err := h.compressFileNolock(f, fileInfo, filePath, compressedFilePath, fileEncoding)
  1198. flock.Unlock()
  1199. return ff, err
  1200. }
  1201. func (h *fsHandler) compressFileNolock(f fs.File, fileInfo fs.FileInfo, filePath, compressedFilePath string, fileEncoding string) (*fsFile, error) {
  1202. // Attempt to open compressed file created by another concurrent
  1203. // goroutine.
  1204. // It is safe opening such a file, since the file creation
  1205. // is guarded by file mutex - see getFileLock call.
  1206. if _, err := os.Stat(compressedFilePath); err == nil {
  1207. _ = f.Close()
  1208. return h.newCompressedFSFile(compressedFilePath, fileEncoding)
  1209. }
  1210. // Create temporary file, so concurrent goroutines don't use
  1211. // it until it is created.
  1212. tmpFilePath := compressedFilePath + ".tmp"
  1213. zf, err := os.Create(tmpFilePath)
  1214. if err != nil {
  1215. _ = f.Close()
  1216. if !errors.Is(err, fs.ErrPermission) {
  1217. return nil, fmt.Errorf("cannot create temporary file %q: %w", tmpFilePath, err)
  1218. }
  1219. return nil, errNoCreatePermission
  1220. }
  1221. if fileEncoding == "br" {
  1222. zw := acquireStacklessBrotliWriter(zf, CompressDefaultCompression)
  1223. _, err = copyZeroAlloc(zw, f)
  1224. if err1 := zw.Flush(); err == nil {
  1225. err = err1
  1226. }
  1227. releaseStacklessBrotliWriter(zw, CompressDefaultCompression)
  1228. } else if fileEncoding == "gzip" {
  1229. zw := acquireStacklessGzipWriter(zf, CompressDefaultCompression)
  1230. _, err = copyZeroAlloc(zw, f)
  1231. if err1 := zw.Flush(); err == nil {
  1232. err = err1
  1233. }
  1234. releaseStacklessGzipWriter(zw, CompressDefaultCompression)
  1235. }
  1236. _ = zf.Close()
  1237. _ = f.Close()
  1238. if err != nil {
  1239. return nil, fmt.Errorf("error when compressing file %q to %q: %w", filePath, tmpFilePath, err)
  1240. }
  1241. if err = os.Chtimes(tmpFilePath, time.Now(), fileInfo.ModTime()); err != nil {
  1242. return nil, fmt.Errorf("cannot change modification time to %v for tmp file %q: %v",
  1243. fileInfo.ModTime(), tmpFilePath, err)
  1244. }
  1245. if err = os.Rename(tmpFilePath, compressedFilePath); err != nil {
  1246. return nil, fmt.Errorf("cannot move compressed file from %q to %q: %w", tmpFilePath, compressedFilePath, err)
  1247. }
  1248. return h.newCompressedFSFile(compressedFilePath, fileEncoding)
  1249. }
  1250. // newCompressedFSFileCache use memory cache compressed files
  1251. func (h *fsHandler) newCompressedFSFileCache(f fs.File, fileInfo fs.FileInfo, filePath, fileEncoding string) (*fsFile, error) {
  1252. var (
  1253. w = &bytebufferpool.ByteBuffer{}
  1254. err error
  1255. )
  1256. if fileEncoding == "br" {
  1257. zw := acquireStacklessBrotliWriter(w, CompressDefaultCompression)
  1258. _, err = copyZeroAlloc(zw, f)
  1259. if err1 := zw.Flush(); err == nil {
  1260. err = err1
  1261. }
  1262. releaseStacklessBrotliWriter(zw, CompressDefaultCompression)
  1263. } else if fileEncoding == "gzip" {
  1264. zw := acquireStacklessGzipWriter(w, CompressDefaultCompression)
  1265. _, err = copyZeroAlloc(zw, f)
  1266. if err1 := zw.Flush(); err == nil {
  1267. err = err1
  1268. }
  1269. releaseStacklessGzipWriter(zw, CompressDefaultCompression)
  1270. }
  1271. defer func() { _ = f.Close() }()
  1272. if err != nil {
  1273. return nil, fmt.Errorf("error when compressing file %q: %w", filePath, err)
  1274. }
  1275. seeker, ok := f.(io.Seeker)
  1276. if !ok {
  1277. return nil, errors.New("not implemented io.Seeker")
  1278. }
  1279. if _, err = seeker.Seek(0, io.SeekStart); err != nil {
  1280. return nil, err
  1281. }
  1282. ext := fileExtension(fileInfo.Name(), false, h.compressedFileSuffixes[fileEncoding])
  1283. contentType := mime.TypeByExtension(ext)
  1284. if len(contentType) == 0 {
  1285. data, err := readFileHeader(f, false, fileEncoding)
  1286. if err != nil {
  1287. return nil, fmt.Errorf("cannot read header of the file %q: %w", fileInfo.Name(), err)
  1288. }
  1289. contentType = http.DetectContentType(data)
  1290. }
  1291. dirIndex := w.B
  1292. lastModified := fileInfo.ModTime()
  1293. ff := &fsFile{
  1294. h: h,
  1295. dirIndex: dirIndex,
  1296. contentType: contentType,
  1297. contentLength: len(dirIndex),
  1298. compressed: true,
  1299. lastModified: lastModified,
  1300. lastModifiedStr: AppendHTTPDate(nil, lastModified),
  1301. t: time.Now(),
  1302. }
  1303. return ff, nil
  1304. }
  1305. func (h *fsHandler) newCompressedFSFile(filePath string, fileEncoding string) (*fsFile, error) {
  1306. f, err := h.filesystem.Open(filePath)
  1307. if err != nil {
  1308. return nil, fmt.Errorf("cannot open compressed file %q: %w", filePath, err)
  1309. }
  1310. fileInfo, err := f.Stat()
  1311. if err != nil {
  1312. _ = f.Close()
  1313. return nil, fmt.Errorf("cannot obtain info for compressed file %q: %w", filePath, err)
  1314. }
  1315. return h.newFSFile(f, fileInfo, true, filePath, fileEncoding)
  1316. }
  1317. func (h *fsHandler) openFSFile(filePath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
  1318. filePathOriginal := filePath
  1319. if mustCompress {
  1320. filePath += h.compressedFileSuffixes[fileEncoding]
  1321. }
  1322. f, err := h.filesystem.Open(filePath)
  1323. if err != nil {
  1324. if mustCompress && errors.Is(err, fs.ErrNotExist) {
  1325. return h.compressAndOpenFSFile(filePathOriginal, fileEncoding)
  1326. }
  1327. return nil, err
  1328. }
  1329. fileInfo, err := f.Stat()
  1330. if err != nil {
  1331. _ = f.Close()
  1332. return nil, fmt.Errorf("cannot obtain info for file %q: %w", filePath, err)
  1333. }
  1334. if fileInfo.IsDir() {
  1335. _ = f.Close()
  1336. if mustCompress {
  1337. return nil, fmt.Errorf("directory with unexpected suffix found: %q. Suffix: %q",
  1338. filePath, h.compressedFileSuffixes[fileEncoding])
  1339. }
  1340. return nil, errDirIndexRequired
  1341. }
  1342. if mustCompress {
  1343. fileInfoOriginal, err := fs.Stat(h.filesystem, filePathOriginal)
  1344. if err != nil {
  1345. _ = f.Close()
  1346. return nil, fmt.Errorf("cannot obtain info for original file %q: %w", filePathOriginal, err)
  1347. }
  1348. // Only re-create the compressed file if there was more than a second between the mod times.
  1349. // On macOS the gzip seems to truncate the nanoseconds in the mod time causing the original file
  1350. // to look newer than the gzipped file.
  1351. if fileInfoOriginal.ModTime().Sub(fileInfo.ModTime()) >= time.Second {
  1352. // The compressed file became stale. Re-create it.
  1353. _ = f.Close()
  1354. _ = os.Remove(filePath)
  1355. return h.compressAndOpenFSFile(filePathOriginal, fileEncoding)
  1356. }
  1357. }
  1358. return h.newFSFile(f, fileInfo, mustCompress, filePath, fileEncoding)
  1359. }
  1360. func (h *fsHandler) newFSFile(f fs.File, fileInfo fs.FileInfo, compressed bool, filePath, fileEncoding string) (*fsFile, error) {
  1361. n := fileInfo.Size()
  1362. contentLength := int(n)
  1363. if n != int64(contentLength) {
  1364. _ = f.Close()
  1365. return nil, fmt.Errorf("too big file: %d bytes", n)
  1366. }
  1367. // detect content-type
  1368. ext := fileExtension(fileInfo.Name(), compressed, h.compressedFileSuffixes[fileEncoding])
  1369. contentType := mime.TypeByExtension(ext)
  1370. if len(contentType) == 0 {
  1371. data, err := readFileHeader(f, compressed, fileEncoding)
  1372. if err != nil {
  1373. return nil, fmt.Errorf("cannot read header of the file %q: %w", fileInfo.Name(), err)
  1374. }
  1375. contentType = http.DetectContentType(data)
  1376. }
  1377. lastModified := fileInfo.ModTime()
  1378. ff := &fsFile{
  1379. h: h,
  1380. f: f,
  1381. filename: filePath,
  1382. contentType: contentType,
  1383. contentLength: contentLength,
  1384. compressed: compressed,
  1385. lastModified: lastModified,
  1386. lastModifiedStr: AppendHTTPDate(nil, lastModified),
  1387. t: time.Now(),
  1388. }
  1389. return ff, nil
  1390. }
  1391. func readFileHeader(f io.Reader, compressed bool, fileEncoding string) ([]byte, error) {
  1392. r := f
  1393. var (
  1394. br *brotli.Reader
  1395. zr *gzip.Reader
  1396. )
  1397. if compressed {
  1398. var err error
  1399. if fileEncoding == "br" {
  1400. if br, err = acquireBrotliReader(f); err != nil {
  1401. return nil, err
  1402. }
  1403. r = br
  1404. } else if fileEncoding == "gzip" {
  1405. if zr, err = acquireGzipReader(f); err != nil {
  1406. return nil, err
  1407. }
  1408. r = zr
  1409. }
  1410. }
  1411. lr := &io.LimitedReader{
  1412. R: r,
  1413. N: 512,
  1414. }
  1415. data, err := io.ReadAll(lr)
  1416. seeker, ok := f.(io.Seeker)
  1417. if !ok {
  1418. return nil, errors.New("must implement io.Seeker")
  1419. }
  1420. if _, err := seeker.Seek(0, io.SeekStart); err != nil {
  1421. return nil, err
  1422. }
  1423. if br != nil {
  1424. releaseBrotliReader(br)
  1425. }
  1426. if zr != nil {
  1427. releaseGzipReader(zr)
  1428. }
  1429. return data, err
  1430. }
  1431. func stripLeadingSlashes(path []byte, stripSlashes int) []byte {
  1432. for stripSlashes > 0 && len(path) > 0 {
  1433. if path[0] != '/' {
  1434. // developer sanity-check
  1435. panic("BUG: path must start with slash")
  1436. }
  1437. n := bytes.IndexByte(path[1:], '/')
  1438. if n < 0 {
  1439. path = path[:0]
  1440. break
  1441. }
  1442. path = path[n+1:]
  1443. stripSlashes--
  1444. }
  1445. return path
  1446. }
  1447. func stripTrailingSlashes(path []byte) []byte {
  1448. for len(path) > 0 && path[len(path)-1] == '/' {
  1449. path = path[:len(path)-1]
  1450. }
  1451. return path
  1452. }
  1453. func fileExtension(path string, compressed bool, compressedFileSuffix string) string {
  1454. if compressed && strings.HasSuffix(path, compressedFileSuffix) {
  1455. path = path[:len(path)-len(compressedFileSuffix)]
  1456. }
  1457. n := strings.LastIndexByte(path, '.')
  1458. if n < 0 {
  1459. return ""
  1460. }
  1461. return path[n:]
  1462. }
  1463. // FileLastModified returns last modified time for the file.
  1464. func FileLastModified(path string) (time.Time, error) {
  1465. f, err := os.Open(path)
  1466. if err != nil {
  1467. return zeroTime, err
  1468. }
  1469. fileInfo, err := f.Stat()
  1470. _ = f.Close()
  1471. if err != nil {
  1472. return zeroTime, err
  1473. }
  1474. return fsModTime(fileInfo.ModTime()), nil
  1475. }
  1476. func fsModTime(t time.Time) time.Time {
  1477. return t.In(time.UTC).Truncate(time.Second)
  1478. }
  1479. var filesLockMap sync.Map
  1480. func getFileLock(absPath string) *sync.Mutex {
  1481. v, _ := filesLockMap.LoadOrStore(absPath, &sync.Mutex{})
  1482. filelock := v.(*sync.Mutex)
  1483. return filelock
  1484. }
  1485. var _ fs.FS = (*osFS)(nil)
  1486. type osFS struct{}
  1487. func (o *osFS) Open(name string) (fs.File, error) { return os.Open(name) }
  1488. func (o *osFS) Stat(name string) (fs.FileInfo, error) { return os.Stat(name) }