Adding docs to hypercore-streams

README.md

@@ -1,6 +1,6 @@
 # hypercore-streams
 
-External implementation of a WriteStream and ReadStream for Hypercore
+Implementation of WriteStream and ReadStream for [hypercore](https://github.com/hypercore-protocol/hypercore).
 
 ```
 npm install hypercore-streams
@@ -11,12 +11,31 @@ npm install hypercore-streams
 ``` js
 const { WriteStream, ReadStream } = require('hypercore-streams')
 
-const ws = new WriteStream(feed)
-const rs = new ReadStream(feed, {
-  start: 0,
-  live: true,
-  valueEncoding: 'json'
-})
+const ws = new WriteStream(feed, [options])
+const rs = new ReadStream(feed, [options])
+```
+
+Options for `WriteStream`:
+
+```js
+{
+  maxBlockSize: Infinity // set this to auto chunk individual blocks if they are larger than this number
+}
+```
+
+Options for `ReadStream`:
+
+```js
+{
+  start: 0, // read from this index
+  end: feed.length, // read until this index
+  snapshot: true, // if set to false it will update `end` to `feed.length` on every read
+  tail: false, // sets `start` to `feed.length`
+  live: false, // set to true to keep reading forever
+  timeout: 0, // timeout for each data event (0 means no timeout)
+  wait: true, // wait for data to be downloaded
+  batch: 1 // amount of messages to read in batch, increasing it (e.g. 100) can improve the performance reading
+}
 ```
 
 ## License

index.js

@@ -42,6 +42,7 @@ class ReadStream extends Readable {
     this.tail = !!opts.tail
     this.index = this.start
     this.options = { wait: opts.wait !== false, ifAvailable: !!opts.ifAvailable, valueEncoding: opts.valueEncoding }
+    this.batch = opts.batch || 1
   }
 
   _open (cb) {
@@ -63,6 +64,21 @@ class ReadStream extends Readable {
       this.push(null)
       return cb(null)
     }
+    if (this.batch > 1) {
+      const batchStart = this.index
+      const batchEnd = Math.min(batchStart + this.batch, this.end, this.feed.length)
+      if (batchStart < batchEnd) {
+        this.index = batchEnd
+        this.feed.getBatch(batchStart, batchEnd, this.options, (err, blocks) => {
+          if (err) return cb(err)
+          for (const block of blocks) {
+            this.push(block)
+          }
+          cb(null)
+        })
+        return
+      }
+    }
     this.feed.get(this.index++, this.options, (err, block) => {
       if (err) return cb(err)
       this.push(block)

package.json

@@ -10,19 +10,20 @@
     "hypercore": "^9.2.1",
     "random-access-memory": "^3.1.1",
     "standard": "^14.3.4",
+    "stream-collector": "^1.0.1",
     "tape": "^5.0.1"
   },
   "scripts": {
-    "test": "standard && tape test.js"
+    "test": "standard && tape test/index.js"
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/mafintosh/hypercore-streams.git"
+    "url": "https://github.com/hypercore-protocol/hypercore-streams.git"
   },
   "author": "Mathias Buus (@mafintosh)",
   "license": "MIT",
   "bugs": {
-    "url": "https://github.com/mafintosh/hypercore-streams/issues"
+    "url": "https://github.com/hypercore-protocol/hypercore-streams/issues"
   },
-  "homepage": "https://github.com/mafintosh/hypercore-streams"
+  "homepage": "https://github.com/hypercore-protocol/hypercore-streams"
 }

test/helpers/create-tracking-ram.js

@@ -0,0 +1,27 @@
+const ram = require('random-access-memory')
+
+module.exports = function () {
+  const logByFilename = {}
+  const factory = function (filename) {
+    const memory = ram()
+    const log = []
+    logByFilename[filename] = log
+    return {
+      read: logAndForward('read'),
+      write: logAndForward('write'),
+      del: logAndForward('del')
+    }
+
+    function logAndForward (op) {
+      return function () {
+        var statement = {}
+        statement[op] = [].slice.apply(arguments)
+        statement[op].pop()
+        log.push(statement)
+        return memory[op].apply(memory, arguments)
+      }
+    }
+  }
+  factory.log = logByFilename
+  return factory
+}