🚧 Added updateAll

feeds/feeds.go

@@ -20,6 +20,9 @@ import (
 	"github.com/mmcdole/gofeed"
 )
 
+// TODO: Test for deadlocks and whether there should be more
+// goroutines for file writing or other things.
+
 var (
 	ErrSaving     = errors.New("couldn't save JSON to disk")
 	ErrNotSuccess = errors.New("status 20 not returned")
@@ -93,11 +96,9 @@ func writeJson() error {
 	enc.SetEscapeHTML(false)
 	enc.SetIndent("", "  ")
 
-	data.feedMu.Lock()
+	data.Lock()
-	data.pageMu.Lock()
 	err = enc.Encode(&data)
-	data.feedMu.Unlock()
+	data.Unlock()
-	data.pageMu.Unlock()
 
 	return err
 }
@@ -191,10 +192,15 @@ func updatePage(url string) error {
 	if _, err := io.Copy(h, res.Body); err != nil {
 		return err
 	}
+	newHash := fmt.Sprintf("%x", h.Sum(nil))
+
 	data.pageMu.Lock()
-	data.Pages[url] = &pageJson{
+	if data.Pages[url].Hash != newHash {
-		Hash:    fmt.Sprintf("%x", h.Sum(nil)),
+		// Page content is different
-		Updated: time.Now().UTC(),
+		data.Pages[url] = &pageJson{
+			Hash:    newHash,
+			Changed: time.Now().UTC(),
+		}
 	}
 	data.pageMu.Unlock()
 
@@ -211,7 +217,62 @@ func updatePage(url string) error {
 }
 
 // updateAll updates all feeds and pages.
-// It should run in goroutine at a regular interval.
 func updateAll() {
+	// TODO: Is two goroutines the right amount?
 
+	worker := func(jobs <-chan [2]string, wg *sync.WaitGroup) {
+		// Each job is: []string{<type>, "url"}
+		// where <type> is "feed" or "page"
+
+		defer wg.Done()
+		for j := range jobs {
+			if j[0] == "feed" {
+				updateFeed(j[1])
+			}
+			if j[0] == "page" {
+				updatePage(j[1])
+			}
+		}
+	}
+
+	var wg sync.WaitGroup
+
+	data.RLock()
+	numJobs := len(data.Feeds) + len(data.Pages)
+	jobs := make(chan [2]string, numJobs)
+
+	// Start 2 workers, waiting for jobs
+	for w := 0; w < 2; w++ {
+		wg.Add(1)
+		go worker(jobs, &wg)
+	}
+
+	// Get map keys in a slice
+
+	feedKeys := make([]string, len(data.Feeds))
+	i := 0
+	for k := range data.Feeds {
+		feedKeys[i] = k
+		i++
+	}
+
+	pageKeys := make([]string, len(data.Pages))
+	i = 0
+	for k := range data.Pages {
+		pageKeys[i] = k
+		i++
+	}
+
+	data.RUnlock()
+
+	for j := 0; j < numJobs; j++ {
+		if j < len(feedKeys) {
+			jobs <- [2]string{"feed", feedKeys[j]}
+		} else {
+			// In the Pages
+			jobs <- [2]string{"page", pageKeys[j-len(feedKeys)]}
+		}
+	}
+
+	wg.Wait()
 }

feeds/structs.go

@@ -17,11 +17,11 @@ Example JSON.
 	"pages": {
 		"url1": {
 			"hash": <hash>,
-			"updated": <time>
+			"changed": <time>
 		},
 		"url2": {
 			"hash": <hash>,
-			"updated": <time>
+			"changed": <time>
 		}
 	}
 }
@@ -39,9 +39,33 @@ type jsonData struct {
 	Pages  map[string]*pageJson    `json:"pages,omitempty"`
 }
 
+// Lock locks both feed and page mutexes.
+func (j *jsonData) Lock() {
+	j.feedMu.Lock()
+	j.pageMu.Lock()
+}
+
+// Unlock unlocks both feed and page mutexes.
+func (j *jsonData) Unlock() {
+	j.feedMu.Unlock()
+	j.pageMu.Unlock()
+}
+
+// RLock read-locks both feed and page mutexes.
+func (j *jsonData) RLock() {
+	j.feedMu.RLock()
+	j.pageMu.RLock()
+}
+
+// RUnlock read-unlocks both feed and page mutexes.
+func (j *jsonData) RUnlock() {
+	j.feedMu.RUnlock()
+	j.pageMu.RUnlock()
+}
+
 type pageJson struct {
 	Hash    string    `json:"hash"`
-	Updated time.Time `json:"updated"`
+	Changed time.Time `json:"changed"` // When the latest change happened
 }
 
 var data jsonData // Global instance of jsonData - loaded from JSON and used