Skip to content

C
caddy
Commit 027f697f authored by Tobias Weingartner's avatar Tobias Weingartner
Browse files
Options

Revamp markdown processing. 

Nuke pre-generation.  This may come back in the form of a more general
caching layer at some later stage.

Nuke index generation.  This should likely be rethought and re-implemented.
parent 6a7b777f
Hide whitespace changes
Inline Side-by-side
Showing with 259 additions and 1088 deletions
caddy/setup/markdown.go
View file @ 027f697f
......@@ -2,9 +2,7 @@ package setup
import (
"net/http"
"path"
"path/filepath"
"strings"
"github.com/mholt/caddy/middleware"
"github.com/mholt/caddy/middleware/markdown"
......@@ -25,25 +23,6 @@ func Markdown(c *Controller) (middleware.Middleware, error) {
IndexFiles: []string{"index.md"},
}
// Sweep the whole path at startup to at least generate link index, maybe generate static site
c.Startup = append(c.Startup, func() error {
for i := range mdconfigs {
cfg := mdconfigs[i]
// Generate link index and static files (if enabled)
if err := markdown.GenerateStatic(md, cfg); err != nil {
return err
}
// Watch file changes for static site generation if not in development mode.
if !cfg.Development {
markdown.Watch(md, cfg, markdown.DefaultInterval)
}
}
return nil
})
return func(next middleware.Handler) middleware.Handler {
md.Next = next
return md
......@@ -55,9 +34,9 @@ func markdownParse(c *Controller) ([]*markdown.Config, error) {
for c.Next() {
md := &markdown.Config{
Renderer: blackfriday.HtmlRenderer(0, "", ""),
Templates: make(map[string]string),
StaticFiles: make(map[string]string),
Renderer: blackfriday.HtmlRenderer(0, "", ""),
Extensions: make(map[string]struct{}),
Templates: make(map[string]string),
}
// Get the path scope
......@@ -80,7 +59,9 @@ func markdownParse(c *Controller) ([]*markdown.Config, error) {
// If no extensions were specified, assume some defaults
if len(md.Extensions) == 0 {
md.Extensions = []string{".md", ".markdown", ".mdown"}
md.Extensions[".md"] = struct{}{}
md.Extensions[".markdown"] = struct{}{}
md.Extensions[".mdown"] = struct{}{}
}
mdconfigs = append(mdconfigs, md)
......@@ -92,11 +73,9 @@ func markdownParse(c *Controller) ([]*markdown.Config, error) {
func loadParams(c *Controller, mdc *markdown.Config) error {
switch c.Val() {
case "ext":
exts := c.RemainingArgs()
if len(exts) == 0 {
return c.ArgErr()
for _, ext := range c.RemainingArgs() {
mdc.Extensions[ext] = struct{}{}
}
mdc.Extensions = append(mdc.Extensions, exts...)
return nil
case "css":
if !c.NextArg() {
......@@ -113,7 +92,7 @@ func loadParams(c *Controller, mdc *markdown.Config) error {
case "template":
tArgs := c.RemainingArgs()
switch len(tArgs) {
case 0:
default:
return c.ArgErr()
case 1:
if _, ok := mdc.Templates[markdown.DefaultTemplate]; ok {
......@@ -126,31 +105,7 @@ func loadParams(c *Controller, mdc *markdown.Config) error {
fpath := filepath.ToSlash(filepath.Clean(c.Root + string(filepath.Separator) + tArgs[1]))
mdc.Templates[tArgs[0]] = fpath
return nil
default:
return c.ArgErr()
}
case "sitegen":
if c.NextArg() {
mdc.StaticDir = path.Join(c.Root, c.Val())
} else {
mdc.StaticDir = path.Join(c.Root, markdown.DefaultStaticDir)
}
if c.NextArg() {
// only 1 argument allowed
return c.ArgErr()
}
return nil
case "dev":
if c.NextArg() {
mdc.Development = strings.ToLower(c.Val()) == "true"
} else {
mdc.Development = true
}
if c.NextArg() {
// only 1 argument allowed
return c.ArgErr()
}
return nil
default:
return c.Err("Expected valid markdown configuration property")
}
......
caddy/setup/markdown_test.go
View file @ 027f697f
package setup
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"testing"
"github.com/mholt/caddy/middleware"
"github.com/mholt/caddy/middleware/markdown"
)
......@@ -37,84 +31,14 @@ func TestMarkdown(t *testing.T) {
if myHandler.Configs[0].PathScope != "/blog" {
t.Errorf("Expected /blog as the Path Scope")
}
if fmt.Sprint(myHandler.Configs[0].Extensions) != fmt.Sprint([]string{".md", ".markdown", ".mdown"}) {
t.Errorf("Expected .md, .markdown, and .mdown as default extensions")
if len(myHandler.Configs[0].Extensions) != 3 {
t.Error("Expected 3 markdown extensions")
}
}
func TestMarkdownStaticGen(t *testing.T) {
c := NewTestController(`markdown /blog {
ext .md
template tpl_with_include.html
sitegen
}`)
c.Root = "./testdata"
mid, err := Markdown(c)
if err != nil {
t.Errorf("Expected no errors, got: %v", err)
}
if mid == nil {
t.Fatal("Expected middleware, was nil instead")
}
for _, start := range c.Startup {
err := start()
if err != nil {
t.Errorf("Startup error: %v", err)
for _, key := range []string{".md", ".markdown", ".mdown"} {
if ext, ok := myHandler.Configs[0].Extensions[key]; !ok {
t.Errorf("Expected extensions to contain %v", ext)
}
}
next := middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) {
t.Fatalf("Next shouldn't be called")
return 0, nil
})
hndlr := mid(next)
mkdwn, ok := hndlr.(markdown.Markdown)
if !ok {
t.Fatalf("Was expecting a markdown.Markdown but got %T", hndlr)
}
expectedStaticFiles := map[string]string{"/blog/first_post.md": "testdata/generated_site/blog/first_post.md/index.html"}
if fmt.Sprint(expectedStaticFiles) != fmt.Sprint(mkdwn.Configs[0].StaticFiles) {
t.Fatalf("Test expected StaticFiles to be %s, but got %s",
fmt.Sprint(expectedStaticFiles), fmt.Sprint(mkdwn.Configs[0].StaticFiles))
}
filePath := "testdata/generated_site/blog/first_post.md/index.html"
if _, err := os.Stat(filePath); err != nil {
t.Fatalf("An error occured when getting the file information: %v", err)
}
html, err := ioutil.ReadFile(filePath)
if err != nil {
t.Fatalf("An error occured when getting the file content: %v", err)
}
expectedBody := []byte(`<!DOCTYPE html>
<html>
<head>
<title>first_post</title>
</head>
<body>
<h1>Header title</h1>
<h1>Test h1</h1>
</body>
</html>
`)
if !bytes.Equal(html, expectedBody) {
t.Fatalf("Expected file content: %s got: %s", string(expectedBody), string(html))
}
fp := filepath.Join(c.Root, markdown.DefaultStaticDir)
if err = os.RemoveAll(fp); err != nil {
t.Errorf("Error while removing the generated static files: %v", err)
}
}
func TestMarkdownParse(t *testing.T) {
......@@ -129,20 +53,23 @@ func TestMarkdownParse(t *testing.T) {
css /resources/css/blog.css
js /resources/js/blog.js
}`, false, []markdown.Config{{
PathScope: "/blog",
Extensions: []string{".md", ".txt"},
Styles: []string{"/resources/css/blog.css"},
Scripts: []string{"/resources/js/blog.js"},
PathScope: "/blog",
Extensions: map[string]struct{}{
".md": struct{}{},
".txt": struct{}{},
},
Styles: []string{"/resources/css/blog.css"},
Scripts: []string{"/resources/js/blog.js"},
}}},
{`markdown /blog {
ext .md
template tpl_with_include.html
sitegen
}`, false, []markdown.Config{{
PathScope: "/blog",
Extensions: []string{".md"},
Templates: map[string]string{markdown.DefaultTemplate: "testdata/tpl_with_include.html"},
StaticDir: markdown.DefaultStaticDir,
PathScope: "/blog",
Extensions: map[string]struct{}{
".md": struct{}{},
},
Templates: map[string]string{markdown.DefaultTemplate: "testdata/tpl_with_include.html"},
}}},
}
for i, test := range tests {
......
middleware/context.go
View file @ 027f697f
......@@ -184,7 +184,11 @@ func (c Context) Markdown(filename string) (string, error) {
return "", err
}
renderer := blackfriday.HtmlRenderer(0, "", "")
extns := blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_STRIKETHROUGH | blackfriday.EXTENSION_DEFINITION_LISTS
extns := 0
extns |= blackfriday.EXTENSION_TABLES
extns |= blackfriday.EXTENSION_FENCED_CODE
extns |= blackfriday.EXTENSION_STRIKETHROUGH
extns |= blackfriday.EXTENSION_DEFINITION_LISTS
markdown := blackfriday.Markdown([]byte(body), renderer, extns)
return string(markdown), nil
......
middleware/markdown/generator.go deleted 100644 → 0
View file @ 6a7b777f
package markdown
import (
"crypto/md5"
"encoding/hex"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"github.com/mholt/caddy/middleware"
)
// GenerateStatic generate static files and link index from markdowns.
// It only generates static files if it is enabled (cfg.StaticDir
// must be set).
func GenerateStatic(md Markdown, cfg *Config) error {
// Generate links since they may be needed, even without sitegen.
generated, err := generateLinks(md, cfg)
if err != nil {
return err
}
// No new file changes, return.
if !generated {
return nil
}
// If static site generation is enabled, generate the site.
if cfg.StaticDir != "" {
if err := generateStaticHTML(md, cfg); err != nil {
return err
}
}
return nil
}
type linkGenerator struct {
gens map[*Config]*linkGen
sync.Mutex
}
var generator = linkGenerator{gens: make(map[*Config]*linkGen)}
// generateLinks generates links to all markdown files ordered by newest date.
// This blocks until link generation is done. When called by multiple goroutines,
// the first caller starts the generation and others only wait.
// It returns if generation is done and any error that occurred.
func generateLinks(md Markdown, cfg *Config) (bool, error) {
generator.Lock()
// if link generator exists for config and running, wait.
if g, ok := generator.gens[cfg]; ok {
if g.started() {
g.addWaiter()
generator.Unlock()
g.Wait()
// another goroutine has done the generation.
return false, g.lastErr
}
}
g := &linkGen{}
generator.gens[cfg] = g
generator.Unlock()
generated := g.generateLinks(md, cfg)
g.discardWaiters()
return generated, g.lastErr
}
// generateStaticHTML generates static HTML files from markdowns.
func generateStaticHTML(md Markdown, cfg *Config) error {
// If generated site already exists, clear it out
_, err := os.Stat(cfg.StaticDir)
if err == nil {
err := os.RemoveAll(cfg.StaticDir)
if err != nil {
return err
}
}
fp := filepath.Join(md.Root, cfg.PathScope)
return filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
for _, ext := range cfg.Extensions {
if !info.IsDir() && strings.HasSuffix(info.Name(), ext) {
// Load the file
body, err := ioutil.ReadFile(path)
if err != nil {
return err
}
// Get the relative path as if it were a HTTP request,
// then prepend with "/" (like a real HTTP request)
reqPath, err := filepath.Rel(md.Root, path)
if err != nil {
return err
}
reqPath = filepath.ToSlash(reqPath)
reqPath = "/" + reqPath
// Create empty requests and url to cater for template values.
req, _ := http.NewRequest("", "/", nil)
urlVar, _ := url.Parse("/")
// Generate the static file
ctx := middleware.Context{Root: md.FileSys, Req: req, URL: urlVar}
_, err = md.Process(cfg, reqPath, body, ctx)
if err != nil {
return err
}
break // don't try other file extensions
}
}
return nil
})
}
// computeDirHash computes an hash on static directory of c.
func computeDirHash(md Markdown, c *Config) (string, error) {
dir := filepath.Join(md.Root, c.PathScope)
if _, err := os.Stat(dir); err != nil {
return "", err
}
hashString := ""
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() && c.IsValidExt(filepath.Ext(path)) {
hashString += fmt.Sprintf("%v%v%v%v", info.ModTime(), info.Name(), info.Size(), path)
}
return nil
})
if err != nil {
return "", err
}
sum := md5.Sum([]byte(hashString))
return hex.EncodeToString(sum[:]), nil
}
middleware/markdown/markdown.go
View file @ 027f697f
......@@ -7,8 +7,7 @@ import (
"log"
"net/http"
"os"
"strings"
"sync"
"path"
"github.com/mholt/caddy/middleware"
"github.com/russross/blackfriday"
......@@ -52,7 +51,7 @@ type Config struct {
PathScope string
// List of extensions to consider as markdown files
Extensions []string
Extensions map[string]struct{}
// List of style sheets to load for each markdown file
Styles []string
......@@ -62,34 +61,6 @@ type Config struct {
// Map of registered templates
Templates map[string]string
// Map of request URL to static files generated
StaticFiles map[string]string
// Links to all markdown pages ordered by date.
Links []PageLink
// Stores a directory hash to check for changes.
linksHash string
// Directory to store static files
StaticDir string
// If in development mode. i.e. Actively editing markdown files.
Development bool
sync.RWMutex
}
// IsValidExt checks to see if an extension is a valid markdown extension
// for config.
func (c *Config) IsValidExt(ext string) bool {
for _, e := range c.Extensions {
if e == ext {
return true
}
}
return false
}
// ServeHTTP implements the http.Handler interface.
......@@ -104,69 +75,39 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
fpath = idx
}
for _, ext := range cfg.Extensions {
if strings.HasSuffix(fpath, ext) {
f, err := md.FileSys.Open(fpath)
if err != nil {
if os.IsPermission(err) {
return http.StatusForbidden, err
}
return http.StatusNotFound, nil
}
fs, err := f.Stat()
if err != nil {
return http.StatusNotFound, nil
}
// if development is set, scan directory for file changes for links.
if cfg.Development {
if err := GenerateStatic(md, cfg); err != nil {
log.Printf("[ERROR] markdown: on-demand site generation error: %v", err)
}
}
cfg.RLock()
filepath, ok := cfg.StaticFiles[fpath]
cfg.RUnlock()
// if static site is generated, attempt to use it
if ok {
if fs1, err := os.Stat(filepath); err == nil {
// if markdown has not been modified since static page
// generation, serve the static page
if fs.ModTime().Before(fs1.ModTime()) {
if html, err := ioutil.ReadFile(filepath); err == nil {
middleware.SetLastModifiedHeader(w, fs1.ModTime())
w.Write(html)
return http.StatusOK, nil
}
if os.IsPermission(err) {
return http.StatusForbidden, err
}
return http.StatusNotFound, nil
}
}
// If supported extension, process it
if _, ok := cfg.Extensions[path.Ext(fpath)]; ok {
f, err := md.FileSys.Open(fpath)
if err != nil {
if os.IsPermission(err) {
return http.StatusForbidden, err
}
return http.StatusNotFound, nil
}
body, err := ioutil.ReadAll(f)
if err != nil {
return http.StatusInternalServerError, err
}
fs, err := f.Stat()
if err != nil {
return http.StatusNotFound, nil
}
ctx := middleware.Context{
Root: md.FileSys,
Req: r,
URL: r.URL,
}
html, err := md.Process(cfg, fpath, body, ctx)
if err != nil {
return http.StatusInternalServerError, err
}
body, err := ioutil.ReadAll(f)
if err != nil {
return http.StatusInternalServerError, err
}
middleware.SetLastModifiedHeader(w, fs.ModTime())
w.Write(html)
return http.StatusOK, nil
ctx := middleware.Context{
Root: md.FileSys,
Req: r,
URL: r.URL,
}
html, err := md.Process(cfg, fpath, body, ctx)
if err != nil {
return http.StatusInternalServerError, err
}
middleware.SetLastModifiedHeader(w, fs.ModTime())
w.Write(html)
return http.StatusOK, nil
}
}
......
middleware/markdown/markdown_test.go
View file @ 027f697f
......@@ -2,12 +2,10 @@ package markdown
import (
"bufio"
"log"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync"
"testing"
"time"
......@@ -23,54 +21,46 @@ func TestMarkdown(t *testing.T) {
FileSys: http.Dir("./testdata"),
Configs: []*Config{
{
Renderer: blackfriday.HtmlRenderer(0, "", ""),
PathScope: "/blog",
Extensions: []string{".md"},
Styles: []string{},
Scripts: []string{},
Templates: templates,
StaticDir: DefaultStaticDir,
StaticFiles: make(map[string]string),
Renderer: blackfriday.HtmlRenderer(0, "", ""),
PathScope: "/blog",
Extensions: map[string]struct{}{
".md": struct{}{},
},
Styles: []string{},
Scripts: []string{},
Templates: templates,
},
{
Renderer: blackfriday.HtmlRenderer(0, "", ""),
PathScope: "/docflags",
Extensions: []string{".md"},
Styles: []string{},
Scripts: []string{},
Renderer: blackfriday.HtmlRenderer(0, "", ""),
PathScope: "/docflags",
Extensions: map[string]struct{}{
".md": struct{}{},
},
Styles: []string{},
Scripts: []string{},
Templates: map[string]string{
DefaultTemplate: "testdata/docflags/template.txt",
},
StaticDir: DefaultStaticDir,
StaticFiles: make(map[string]string),
},
{
Renderer: blackfriday.HtmlRenderer(0, "", ""),
PathScope: "/log",
Extensions: []string{".md"},
Styles: []string{"/resources/css/log.css", "/resources/css/default.css"},
Scripts: []string{"/resources/js/log.js", "/resources/js/default.js"},
Templates: make(map[string]string),
StaticDir: DefaultStaticDir,
StaticFiles: make(map[string]string),
Renderer: blackfriday.HtmlRenderer(0, "", ""),
PathScope: "/log",
Extensions: map[string]struct{}{
".md": struct{}{},
},
Styles: []string{"/resources/css/log.css", "/resources/css/default.css"},
Scripts: []string{"/resources/js/log.js", "/resources/js/default.js"},
Templates: make(map[string]string),
},
{
Renderer: blackfriday.HtmlRenderer(0, "", ""),
PathScope: "/og",
Extensions: []string{".md"},
Styles: []string{},
Scripts: []string{},
Templates: templates,
StaticDir: "testdata/og_static",
StaticFiles: map[string]string{"/og/first.md": "testdata/og_static/og/first.md/index.html"},
Links: []PageLink{
{
Title: "first",
Summary: "",
Date: time.Now(),
URL: "/og/first.md",
},
Renderer: blackfriday.HtmlRenderer(0, "", ""),
PathScope: "/og",
Extensions: map[string]struct{}{
".md": struct{}{},
},
Styles: []string{},
Scripts: []string{},
Templates: templates,
},
},
IndexFiles: []string{"index.html"},
......@@ -80,14 +70,6 @@ func TestMarkdown(t *testing.T) {
}),
}
for i := range md.Configs {
c := md.Configs[i]
if err := GenerateStatic(md, c); err != nil {
t.Fatalf("Error: %v", err)
}
Watch(md, c, time.Millisecond*100)
}
req, err := http.NewRequest("GET", "/blog/test.md", nil)
if err != nil {
t.Fatalf("Could not create HTTP request: %v", err)
......@@ -219,52 +201,6 @@ Welcome to title!
if !equalStrings(respBody, expectedBody) {
t.Fatalf("Expected body: %v got: %v", expectedBody, respBody)
}
expectedLinks := []string{
"/blog/test.md",
"/docflags/test.md",
"/log/test.md",
}
for i, c := range md.Configs[:2] {
log.Printf("Test number: %d, configuration links: %v, config: %v", i, c.Links, c)
if c.Links[0].URL != expectedLinks[i] {
t.Fatalf("Expected %v got %v", expectedLinks[i], c.Links[0].URL)
}
}
// attempt to trigger race conditions
var w sync.WaitGroup
f := func() {
req, err := http.NewRequest("GET", "/log/test.md", nil)
if err != nil {
t.Fatalf("Could not create HTTP request: %v", err)
}
rec := httptest.NewRecorder()
md.ServeHTTP(rec, req)
w.Done()
}
for i := 0; i < 5; i++ {
w.Add(1)
go f()
}
w.Wait()
f = func() {
GenerateStatic(md, md.Configs[0])
w.Done()
}
for i := 0; i < 5; i++ {
w.Add(1)
go f()
}
w.Wait()
if err = os.RemoveAll(DefaultStaticDir); err != nil {
t.Errorf("Error while removing the generated static files: %v", err)
}
}
func equalStrings(s1, s2 string) bool {
......
middleware/markdown/metadata.go
View file @ 027f697f
......@@ -2,12 +2,16 @@ package markdown
import (
"bytes"
"encoding/json"
"fmt"
"time"
)
"github.com/BurntSushi/toml"
"gopkg.in/yaml.v2"
var (
// Date format YYYY-MM-DD HH:MM:SS or YYYY-MM-DD
timeLayout = []string{
`2006-01-02 15:04:05`,
`2006-01-02`,
}
)
// Metadata stores a page's metadata
......@@ -30,6 +34,8 @@ type Metadata struct {
// load loads parsed values in parsedMap into Metadata
func (m *Metadata) load(parsedMap map[string]interface{}) {
// Pull top level things out
if title, ok := parsedMap["title"]; ok {
m.Title, _ = title.(string)
}
......@@ -37,17 +43,21 @@ func (m *Metadata) load(parsedMap map[string]interface{}) {
m.Template, _ = template.(string)
}
if date, ok := parsedMap["date"].(string); ok {
if t, err := time.Parse(timeLayout, date); err == nil {
m.Date = t
for _, layout := range timeLayout {
if t, err := time.Parse(layout, date); err == nil {
m.Date = t
break
}
}
}
// store everything as a variable
// Store everything as a flag or variable
for key, val := range parsedMap {
switch v := val.(type) {
case string:
m.Variables[key] = v
case bool:
m.Flags[key] = v
case string:
m.Variables[key] = v
}
}
}
......@@ -70,116 +80,6 @@ type MetadataParser interface {
Metadata() Metadata
}
// JSONMetadataParser is the MetadataParser for JSON
type JSONMetadataParser struct {
metadata Metadata
}
// Parse the metadata
func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) {
b, markdown, err := extractMetadata(j, b)
if err != nil {
return markdown, err
}
m := make(map[string]interface{})
// Read the preceding JSON object
decoder := json.NewDecoder(bytes.NewReader(b))
if err := decoder.Decode(&m); err != nil {
return markdown, err
}
j.metadata.load(m)
return markdown, nil
}
// Metadata returns parsed metadata. It should be called
// only after a call to Parse returns without error.
func (j *JSONMetadataParser) Metadata() Metadata {
return j.metadata
}
// Opening returns the opening identifier JSON metadata
func (j *JSONMetadataParser) Opening() []byte {
return []byte("{")
}
// Closing returns the closing identifier JSON metadata
func (j *JSONMetadataParser) Closing() []byte {
return []byte("}")
}
// TOMLMetadataParser is the MetadataParser for TOML
type TOMLMetadataParser struct {
metadata Metadata
}
// Parse the metadata
func (t *TOMLMetadataParser) Parse(b []byte) ([]byte, error) {
b, markdown, err := extractMetadata(t, b)
if err != nil {
return markdown, err
}
m := make(map[string]interface{})
if err := toml.Unmarshal(b, &m); err != nil {
return markdown, err
}
t.metadata.load(m)
return markdown, nil
}
// Metadata returns parsed metadata. It should be called
// only after a call to Parse returns without error.
func (t *TOMLMetadataParser) Metadata() Metadata {
return t.metadata
}
// Opening returns the opening identifier TOML metadata
func (t *TOMLMetadataParser) Opening() []byte {
return []byte("+++")
}
// Closing returns the closing identifier TOML metadata
func (t *TOMLMetadataParser) Closing() []byte {
return []byte("+++")
}
// YAMLMetadataParser is the MetadataParser for YAML
type YAMLMetadataParser struct {
metadata Metadata
}
// Parse the metadata
func (y *YAMLMetadataParser) Parse(b []byte) ([]byte, error) {
b, markdown, err := extractMetadata(y, b)
if err != nil {
return markdown, err
}
m := make(map[string]interface{})
if err := yaml.Unmarshal(b, &m); err != nil {
return markdown, err
}
y.metadata.load(m)
return markdown, nil
}
// Metadata returns parsed metadata. It should be called
// only after a call to Parse returns without error.
func (y *YAMLMetadataParser) Metadata() Metadata {
return y.metadata
}
// Opening returns the opening identifier YAML metadata
func (y *YAMLMetadataParser) Opening() []byte {
return []byte("---")
}
// Closing returns the closing identifier YAML metadata
func (y *YAMLMetadataParser) Closing() []byte {
return []byte("---")
}
// extractMetadata separates metadata content from from markdown content in b.
// It returns the metadata, the remaining bytes (markdown), and an error, if any.
func extractMetadata(parser MetadataParser, b []byte) (metadata []byte, markdown []byte, err error) {
......
middleware/markdown/metadata_json.go 0 → 100644
View file @ 027f697f
package markdown
import (
"bytes"
"encoding/json"
)
// JSONMetadataParser is the MetadataParser for JSON
type JSONMetadataParser struct {
metadata Metadata
}
// Parse the metadata
func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) {
b, markdown, err := extractMetadata(j, b)
if err != nil {
return markdown, err
}
m := make(map[string]interface{})
// Read the preceding JSON object
decoder := json.NewDecoder(bytes.NewReader(b))
if err := decoder.Decode(&m); err != nil {
return markdown, err
}
j.metadata.load(m)
return markdown, nil
}
// Metadata returns parsed metadata. It should be called
// only after a call to Parse returns without error.
func (j *JSONMetadataParser) Metadata() Metadata {
return j.metadata
}
// Opening returns the opening identifier JSON metadata
func (j *JSONMetadataParser) Opening() []byte {
return []byte("{")
}
// Closing returns the closing identifier JSON metadata
func (j *JSONMetadataParser) Closing() []byte {
return []byte("}")
}
middleware/markdown/metadata_toml.go 0 → 100644
View file @ 027f697f
package markdown
import (
"github.com/BurntSushi/toml"
)
// TOMLMetadataParser is the MetadataParser for TOML
type TOMLMetadataParser struct {
metadata Metadata
}
// Parse the metadata
func (t *TOMLMetadataParser) Parse(b []byte) ([]byte, error) {
b, markdown, err := extractMetadata(t, b)
if err != nil {
return markdown, err
}
m := make(map[string]interface{})
if err := toml.Unmarshal(b, &m); err != nil {
return markdown, err
}
t.metadata.load(m)
return markdown, nil
}
// Metadata returns parsed metadata. It should be called
// only after a call to Parse returns without error.
func (t *TOMLMetadataParser) Metadata() Metadata {
return t.metadata
}
// Opening returns the opening identifier TOML metadata
func (t *TOMLMetadataParser) Opening() []byte {
return []byte("+++")
}
// Closing returns the closing identifier TOML metadata
func (t *TOMLMetadataParser) Closing() []byte {
return []byte("+++")
}
middleware/markdown/metadata_yaml.go 0 → 100644
View file @ 027f697f
package markdown
import (
"gopkg.in/yaml.v2"
)
// YAMLMetadataParser is the MetadataParser for YAML
type YAMLMetadataParser struct {
metadata Metadata
}
// Parse the metadata
func (y *YAMLMetadataParser) Parse(b []byte) ([]byte, error) {
b, markdown, err := extractMetadata(y, b)
if err != nil {
return markdown, err
}
m := make(map[string]interface{})
if err := yaml.Unmarshal(b, &m); err != nil {
return markdown, err
}
y.metadata.load(m)
return markdown, nil
}
// Metadata returns parsed metadata. It should be called
// only after a call to Parse returns without error.
func (y *YAMLMetadataParser) Metadata() Metadata {
return y.metadata
}
// Opening returns the opening identifier YAML metadata
func (y *YAMLMetadataParser) Opening() []byte {
return []byte("---")
}
// Closing returns the closing identifier YAML metadata
func (y *YAMLMetadataParser) Closing() []byte {
return []byte("---")
}
middleware/markdown/page.go deleted 100644 → 0
View file @ 6a7b777f
package markdown
import (
"bytes"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/russross/blackfriday"
)
const (
// Date format YYYY-MM-DD HH:MM:SS
timeLayout = `2006-01-02 15:04:05`
// Maximum length of page summary.
summaryLen = 500
)
// PageLink represents a statically generated markdown page.
type PageLink struct {
Title string
Summary string
Date time.Time
URL string
}
// byDate sorts PageLink by newest date to oldest.
type byDate []PageLink
func (p byDate) Len() int { return len(p) }
func (p byDate) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p byDate) Less(i, j int) bool { return p[i].Date.After(p[j].Date) }
type linkGen struct {
generating bool
waiters int
lastErr error
sync.RWMutex
sync.WaitGroup
}
func (l *linkGen) addWaiter() {
l.WaitGroup.Add(1)
l.waiters++
}
func (l *linkGen) discardWaiters() {
l.Lock()
defer l.Unlock()
for i := 0; i < l.waiters; i++ {
l.Done()
}
}
func (l *linkGen) started() bool {
l.RLock()
defer l.RUnlock()
return l.generating
}
// generateLinks generate links to markdown files if there are file changes.
// It returns true when generation is done and false otherwise.
func (l *linkGen) generateLinks(md Markdown, cfg *Config) bool {
l.Lock()
l.generating = true
l.Unlock()
fp := filepath.Join(md.Root, cfg.PathScope) // path to scan for .md files
// If the file path to scan for Markdown files (fp) does
// not exist, there are no markdown files to scan for.
if _, err := os.Stat(fp); os.IsNotExist(err) {
l.Lock()
l.lastErr = err
l.generating = false
l.Unlock()
return false
}
hash, err := computeDirHash(md, cfg)
// same hash, return.
if err == nil && hash == cfg.linksHash {
l.Lock()
l.generating = false
l.Unlock()
return false
} else if err != nil {
log.Printf("[ERROR] markdown: Hash error: %v", err)
}
cfg.Links = []PageLink{}
cfg.Lock()
l.lastErr = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
for _, ext := range cfg.Extensions {
if info.IsDir() || !strings.HasSuffix(info.Name(), ext) {
continue
}
// Load the file
body, err := ioutil.ReadFile(path)
if err != nil {
return err
}
// Get the relative path as if it were a HTTP request,
// then prepend with "/" (like a real HTTP request)
reqPath, err := filepath.Rel(md.Root, path)
if err != nil {
return err
}
reqPath = "/" + filepath.ToSlash(reqPath)
// Make the summary
parser := findParser(body)
if parser == nil {
// no metadata, ignore.
continue
}
summaryRaw, err := parser.Parse(body)
if err != nil {
return err
}
summary := blackfriday.Markdown(summaryRaw, SummaryRenderer{}, 0)
// truncate summary to maximum length
if len(summary) > summaryLen {
summary = summary[:summaryLen]
// trim to nearest word
lastSpace := bytes.LastIndex(summary, []byte(" "))
if lastSpace != -1 {
summary = summary[:lastSpace]
}
}
metadata := parser.Metadata()
cfg.Links = append(cfg.Links, PageLink{
Title: metadata.Title,
URL: reqPath,
Date: metadata.Date,
Summary: string(summary),
})
break // don't try other file extensions
}
return nil
})
// sort by newest date
sort.Sort(byDate(cfg.Links))
cfg.linksHash = hash
cfg.Unlock()
l.Lock()
l.generating = false
l.Unlock()
return true
}
middleware/markdown/process.go
View file @ 027f697f
......@@ -3,10 +3,7 @@ package markdown
import (
"bytes"
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
"text/template"
"github.com/mholt/caddy/middleware"
......@@ -16,8 +13,6 @@ import (
const (
// DefaultTemplate is the default template.
DefaultTemplate = "defaultTemplate"
// DefaultStaticDir is the default static directory.
DefaultStaticDir = "generated_site"
)
// Data represents a markdown document.
......@@ -25,7 +20,8 @@ type Data struct {
middleware.Context
Doc map[string]string
DocFlags map[string]bool
Links []PageLink
Styles []string
Scripts []string
}
// Include "overrides" the embedded middleware.Context's Include()
......@@ -75,7 +71,11 @@ func (md Markdown) Process(c *Config, requestPath string, b []byte, ctx middlewa
}
// process markdown
extns := blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_STRIKETHROUGH | blackfriday.EXTENSION_DEFINITION_LISTS
extns := 0
extns |= blackfriday.EXTENSION_TABLES
extns |= blackfriday.EXTENSION_FENCED_CODE
extns |= blackfriday.EXTENSION_STRIKETHROUGH
extns |= blackfriday.EXTENSION_DEFINITION_LISTS
markdown = blackfriday.Markdown(markdown, c.Renderer, extns)
// set it as body for template
......@@ -94,123 +94,51 @@ func (md Markdown) Process(c *Config, requestPath string, b []byte, ctx middlewa
// processTemplate processes a template given a requestPath,
// template (tmpl) and metadata
func (md Markdown) processTemplate(c *Config, requestPath string, tmpl []byte, metadata Metadata, ctx middleware.Context) ([]byte, error) {
var t *template.Template
var err error
// if template is not specified,
// use the default template
if tmpl == nil {
tmpl = defaultTemplate(c, metadata, requestPath)
t = template.Must(template.New("").Parse(htmlTemplate))
} else {
t, err = template.New("").Parse(string(tmpl))
if err != nil {
return nil, err
}
}
// process the template
b := new(bytes.Buffer)
t, err := template.New("").Parse(string(tmpl))
if err != nil {
return nil, err
}
mdData := Data{
Context: ctx,
Doc: metadata.Variables,
DocFlags: metadata.Flags,
Links: c.Links,
Styles: c.Styles,
Scripts: c.Scripts,
}
c.RLock()
b := new(bytes.Buffer)
err = t.Execute(b, mdData)
c.RUnlock()
if err != nil {
return nil, err
}
// generate static page
if err = md.generatePage(c, requestPath, b.Bytes()); err != nil {
// if static page generation fails, nothing fatal, only log the error.
// TODO: Report (return) this non-fatal error, but don't log it here?
log.Println("[ERROR] markdown: Render:", err)
}
return b.Bytes(), nil
}
// generatePage generates a static html page from the markdown in content if c.StaticDir
// is a non-empty value, meaning that the user enabled static site generation.
func (md Markdown) generatePage(c *Config, requestPath string, content []byte) error {
// Only generate the page if static site generation is enabled
if c.StaticDir != "" {
// if static directory is not existing, create it
if _, err := os.Stat(c.StaticDir); err != nil {
err := os.MkdirAll(c.StaticDir, os.FileMode(0755))
if err != nil {
return err
}
}
// the URL will always use "/" as a path separator,
// convert that to a native path to support OS that
// use different path separators
filePath := filepath.Join(c.StaticDir, filepath.FromSlash(requestPath))
// If it is index file, use the directory instead
if md.IsIndexFile(filepath.Base(requestPath)) {
filePath, _ = filepath.Split(filePath)
}
// Create the directory in case it is not existing
if err := os.MkdirAll(filePath, os.FileMode(0744)); err != nil {
return err
}
// generate index.html file in the directory
filePath = filepath.Join(filePath, "index.html")
err := ioutil.WriteFile(filePath, content, os.FileMode(0664))
if err != nil {
return err
}
c.Lock()
c.StaticFiles[requestPath] = filepath.ToSlash(filePath)
c.Unlock()
}
return nil
}
// defaultTemplate constructs a default template.
func defaultTemplate(c *Config, metadata Metadata, requestPath string) []byte {
var scripts, styles bytes.Buffer
for _, style := range c.Styles {
styles.WriteString(strings.Replace(cssTemplate, "{{url}}", style, 1))
styles.WriteString("\r\n")
}
for _, script := range c.Scripts {
scripts.WriteString(strings.Replace(jsTemplate, "{{url}}", script, 1))
scripts.WriteString("\r\n")
}
// Title is first line (length-limited), otherwise filename
title, _ := metadata.Variables["title"]
html := []byte(htmlTemplate)
html = bytes.Replace(html, []byte("{{title}}"), []byte(title), 1)
html = bytes.Replace(html, []byte("{{css}}"), styles.Bytes(), 1)
html = bytes.Replace(html, []byte("{{js}}"), scripts.Bytes(), 1)
return html
}
const (
htmlTemplate = `<!DOCTYPE html>
<html>
<head>
<title>{{title}}</title>
<title>{{.Doc.title}}</title>
<meta charset="utf-8">
{{css}}
{{js}}
{{range .Styles}}<link rel="stylesheet" href="{{.}}">
{{end -}}
{{range .Scripts}}<script src="{{.}}"></script>
{{end -}}
</head>
<body>
{{.Doc.body}}
</body>
</html>`
cssTemplate = `<link rel="stylesheet" href="{{url}}">`
jsTemplate = `<script src="{{url}}"></script>`
)
middleware/markdown/renderer.go deleted 100644 → 0
View file @ 6a7b777f
package markdown
import (
"bytes"
)
// SummaryRenderer represents a summary renderer.
type SummaryRenderer struct{}
// Block-level callbacks
// BlockCode is the code tag callback.
func (r SummaryRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {}
// BlockQuote is the quote tag callback.
func (r SummaryRenderer) BlockQuote(out *bytes.Buffer, text []byte) {}
// BlockHtml is the HTML tag callback.
func (r SummaryRenderer) BlockHtml(out *bytes.Buffer, text []byte) {}
// Header is the header tag callback.
func (r SummaryRenderer) Header(out *bytes.Buffer, text func() bool, level int, id string) {}
// HRule is the horizontal rule tag callback.
func (r SummaryRenderer) HRule(out *bytes.Buffer) {}
// List is the list tag callback.
func (r SummaryRenderer) List(out *bytes.Buffer, text func() bool, flags int) {
// TODO: This is not desired (we'd rather not write lists as part of summary),
// but see this issue: https://github.com/russross/blackfriday/issues/189
marker := out.Len()
if !text() {
out.Truncate(marker)
}
out.Write([]byte{' '})
}
// ListItem is the list item tag callback.
func (r SummaryRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {}
// Paragraph is the paragraph tag callback.
func (r SummaryRenderer) Paragraph(out *bytes.Buffer, text func() bool) {
marker := out.Len()
if !text() {
out.Truncate(marker)
}
out.Write([]byte{' '})
}
// Table is the table tag callback.
func (r SummaryRenderer) Table(out *bytes.Buffer, header []byte, body []byte, columnData []int) {}
// TableRow is the table row tag callback.
func (r SummaryRenderer) TableRow(out *bytes.Buffer, text []byte) {}
// TableHeaderCell is the table header cell tag callback.
func (r SummaryRenderer) TableHeaderCell(out *bytes.Buffer, text []byte, flags int) {}
// TableCell is the table cell tag callback.
func (r SummaryRenderer) TableCell(out *bytes.Buffer, text []byte, flags int) {}
// Footnotes is the foot notes tag callback.
func (r SummaryRenderer) Footnotes(out *bytes.Buffer, text func() bool) {}
// FootnoteItem is the footnote item tag callback.
func (r SummaryRenderer) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) {}
// TitleBlock is the title tag callback.
func (r SummaryRenderer) TitleBlock(out *bytes.Buffer, text []byte) {}
// Span-level callbacks
// AutoLink is the autolink tag callback.
func (r SummaryRenderer) AutoLink(out *bytes.Buffer, link []byte, kind int) {}
// CodeSpan is the code span tag callback.
func (r SummaryRenderer) CodeSpan(out *bytes.Buffer, text []byte) {
out.Write([]byte("`"))
out.Write(text)
out.Write([]byte("`"))
}
// DoubleEmphasis is the double emphasis tag callback.
func (r SummaryRenderer) DoubleEmphasis(out *bytes.Buffer, text []byte) {
out.Write(text)
}
// Emphasis is the emphasis tag callback.
func (r SummaryRenderer) Emphasis(out *bytes.Buffer, text []byte) {
out.Write(text)
}
// Image is the image tag callback.
func (r SummaryRenderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {}
// LineBreak is the line break tag callback.
func (r SummaryRenderer) LineBreak(out *bytes.Buffer) {}
// Link is the link tag callback.
func (r SummaryRenderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
out.Write(content)
}
// RawHtmlTag is the raw HTML tag callback.
func (r SummaryRenderer) RawHtmlTag(out *bytes.Buffer, tag []byte) {}
// TripleEmphasis is the triple emphasis tag callback.
func (r SummaryRenderer) TripleEmphasis(out *bytes.Buffer, text []byte) {
out.Write(text)
}
// StrikeThrough is the strikethrough tag callback.
func (r SummaryRenderer) StrikeThrough(out *bytes.Buffer, text []byte) {}
// FootnoteRef is the footnote ref tag callback.
func (r SummaryRenderer) FootnoteRef(out *bytes.Buffer, ref []byte, id int) {}
// Low-level callbacks
// Entity callback.
func (r SummaryRenderer) Entity(out *bytes.Buffer, entity []byte) {
out.Write(entity)
}
// NormalText callback.
func (r SummaryRenderer) NormalText(out *bytes.Buffer, text []byte) {
out.Write(text)
}
// Header and footer
// DocumentHeader callback.
func (r SummaryRenderer) DocumentHeader(out *bytes.Buffer) {}
// DocumentFooter callback.
func (r SummaryRenderer) DocumentFooter(out *bytes.Buffer) {}
// GetFlags returns zero.
func (r SummaryRenderer) GetFlags() int { return 0 }
middleware/markdown/watcher.go deleted 100644 → 0
View file @ 6a7b777f
package markdown
import (
"log"
"time"
)
// DefaultInterval is the default interval at which the markdown watcher
// checks for changes.
const DefaultInterval = time.Second * 60
// Watch monitors the configured markdown directory for changes. It calls GenerateLinks
// when there are changes.
func Watch(md Markdown, c *Config, interval time.Duration) (stopChan chan struct{}) {
return TickerFunc(interval, func() {
if err := GenerateStatic(md, c); err != nil {
log.Printf("[ERROR] markdown: Re-generating static site: %v", err)
}
})
}
// TickerFunc runs f at interval. A message to the returned channel will stop the
// executing goroutine.
func TickerFunc(interval time.Duration, f func()) chan struct{} {
stopChan := make(chan struct{})
ticker := time.NewTicker(interval)
go func() {
loop:
for {
select {
case <-ticker.C:
f()
case <-stopChan:
ticker.Stop()
break loop
}
}
}()
return stopChan
}
middleware/markdown/watcher_test.go deleted 100644 → 0
View file @ 6a7b777f
package markdown
import (
"fmt"
"strings"
"sync"
"testing"
"time"
)
func TestWatcher(t *testing.T) {
expected := "12345678"
interval := time.Millisecond * 100
i := 0
out := ""
syncChan := make(chan struct{})
stopChan := TickerFunc(interval, func() {
i++
out += fmt.Sprint(i)
syncChan <- struct{}{}
})
sleepInSync(8, syncChan, stopChan)
if out != expected {
t.Fatalf("Expected to have prefix %v, found %v", expected, out)
}
out = ""
i = 0
var mu sync.Mutex
stopChan = TickerFunc(interval, func() {
i++
mu.Lock()
out += fmt.Sprint(i)
mu.Unlock()
syncChan <- struct{}{}
})
sleepInSync(9, syncChan, stopChan)
mu.Lock()
res := out
mu.Unlock()
if !strings.HasPrefix(res, expected) || res == expected {
t.Fatalf("expected (%v) must be a proper prefix of out(%v).", expected, out)
}
}
func sleepInSync(times int, syncChan chan struct{}, stopChan chan struct{}) {
for i := 0; i < times; i++ {
<-syncChan
}
stopChan <- struct{}{}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7