Wrapping Up the Word Count Spider

require 'yaml'
require 'json'
require_relative 'options'
require_relative 'crawler'

module Tentacles
  class Runner

    def initialize(config)
      @options = Tentacles::Options.new(config)
      @path = File.dirname(__FILE__) + '/../../output/'
      @filename = 'word_count.json'
    end

    def run
      @crawler = Tentacles::Crawler.from_uri(@options.uri)
      output = @crawler.words_by_selector(@options.post_selector, @options.ignored_post_selector)

      Dir.mkdir(@path) unless Dir.exists?(@path)

      File.open(@path + @filename, "w") do |file|
        file.puts JSON.pretty_generate(output)
      end
    end
  end
end

require 'open-uri'
require 'nokogiri'
require 'mechanize'

module Tentacles
  class Crawler

    attr_reader :doc

    def self.from_uri(uri)
      new(uri)
    end

    def initialize(uri)
      # Create a new instance of Mechanize and grab our page
      @agent = Mechanize.new

      @uri = uri
      @page = @agent.get(@uri)
      @counts = Hash.new(0)
    end

    def words_by_selector(selector, ignored_selector = nil)
      # Get all the links on the page
      post_links = @page.links.find_all { |l| l.attributes.parent.name == 'h1' }
      post_links.shift # Get rid of the first anchor since it's the site header
      post_links.each do |link|
        post = link.click
        @doc = post.parser
        nodes = nodes_by_selector(selector)
        nodes.each do |node|
          if ignored_selector
            ignored = node.css(ignored_selector)
            ignored.remove()
          end
          words = words_from_string(node.content)
          count_frequency(words)
        end
      end

      sorted = @counts.sort_by { |word, count| count }
      sorted.reverse!
      sorted.map! do |word, count|
        { word: word, count: count }
      end
      { word_count: sorted }
    end

    def metadata_by_selector(selector)
      node = nodes_by_selector(selector).first
      metadata = {}
      node.children.each do |child|
        child.content
      end
    end

  private

    def nodes_by_selector(selector)
      nodes = @doc.css(selector)
      raise Tentacles::SelectionError,
        'The selector did not return an results!' if nodes.empty?
      nodes
    end

    def words_from_string(string)
      string.downcase.scan(/[\w']+/)
    end

    def count_frequency(word_list)
      for word in word_list
        @counts[word] += 1
      end
      @counts
    end
  end
end

require_relative '../lib/tentacles/runner'
require 'helpers'
require 'fakeweb'

describe Tentacles::Runner do
  include Helpers

  before do
    @runner = Tentacles::Runner.new(relative_path + '/../lib/tentacles/config.yml')

    # Create a mock options object
    @options = {
      uri: 'http://robdodson.me/blog/archives',
      post_selector: '.entry-content',
      ignored_post_selector: 'ul:last-child',
      metadata_selector: '.entry-content ul:last-child'
    }
    @path = File.dirname(__FILE__) + '/../output/'
    @filename = 'word_count.json'
  end

  subject { @runner }

  it { should respond_to(:run) }

  describe "when parsing the config file" do
    it "should raise an error if the config file is missing" do
      expect { runner = Tentacles::Runner.new('') }.to raise_error(Errno::ENOENT)
      expect { runner = Tentacles::Runner.new(nil) }.to raise_error(TypeError)
    end

    it "should raise an error if the config file is invalid" do
      expect { runner = Tentacles::Runner.new(relative_path + '/mocks/invalid_yaml.yml') }.to raise_error(Psych::SyntaxError)
    end

    it "should create a directory for our output" do
      @runner.run
      Dir.exists?(@path).should be_true
    end

    it "should output the correct JSON" do
      @runner.run
      File.open(@path + @filename) do |file|
        file.each_line do |line|
          puts line
        end
      end
    end
  end
end

{
  "word_count": [
    {
      "word": "the",
      "count": 1678
    },
    {
      "word": "to",
      "count": 1548
    },
    {
      "word": "a",
      "count": 1023
    },
    {
      "word": "i",
      "count": 792
    },
    {
      "word": "it",
      "count": 730
    },
    {
      "word": "and",
      "count": 718
    },
    {
      "word": "this",
      "count": 661
    },
    {
      "word": "of",
      "count": 658
    },
    {
      "word": "you",
      "count": 640
    },
    {
      "word": "that",
      "count": 585
    },
    {
      "word": "we",
      "count": 569
    }
... .