diff --git a/Gemfile.lock b/Gemfile.lock index 62471b11..710795b8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -84,7 +84,7 @@ GEM mime-types mimemagic (~> 0.3.0) terrapin (~> 0.6.0) - rack (2.0.6) + rack (2.0.8) rack-test (1.1.0) rack (>= 1.0, < 3) rails (5.2.2) diff --git a/README.md b/README.md index a36f3ba9..9c4c829d 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,33 @@ This won't do you much good if you don't have the data, but this is my attempt t It took too long, but it works. -I'll be putting the sql export up at some point so you can play with this yourself, and maybe make it prettier, more responsive, etc. +## Local Setup + +### Requirements + +* Ruby 2.6.3 +* MySQL or MariaDB + * The default database.yml is localhost, and root w/out a password. + * The database name is `ficly_export` + +### Getting the Data + +The sql file to import is in db/export.sql.gz. It's a little over 600mb uncompressed. So, you'll need to do something like: + +```cd db +gunzip export.sql.gz +mysql < export.sql +``` + +### Setting up Rails + +* You'll need to install Ruby 2.6.3 however you want to make that happen. +* Install the bundler gem: `gem install bundler` +* `bundle` + +And there you go! + +## TODO + +* Could be better looking, especially on mobile. You're welcome to help with that and submit a pull request! +* Speed up the export process. You're also welcome to help with that. To look at the monster as it is, check out lib/tasks/export.rake. diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb index 8b37aeb6..26bf1a19 100644 --- a/app/controllers/tags_controller.rb +++ b/app/controllers/tags_controller.rb @@ -3,15 +3,15 @@ class TagsController < ApplicationController def index @page = params[:page].to_i @page = 1 if @page < 1 - @tags = Tag.where.not(cleaned_tag: [nil, "", '-']).order('stories_count desc, cleaned_tag asc').paginate(page: @page, per_page: 250) + @tags = Tag.where.not(cleaned_tag: [nil, "", '-']).where("stories_count > 0").order('stories_count desc, cleaned_tag asc').paginate(page: @page, per_page: 250) end def show @tag = Tag.find_by(cleaned_tag: params[:id]) story_ids = @tag.taggings.where(taggable_type: "Story").pluck(:taggable_id) @stories = Story.published.where(id: story_ids).order("published_at asc").paginate(page: 1, per_page: 50) - challenge_ids = @tag.taggings.where(taggable_type: "Challenge").pluck(:taggable_id) - @challenges = Challenge.where(id: challenge_ids).order("id asc").paginate(page: 1, per_page: 50) + # challenge_ids = @tag.taggings.where(taggable_type: "Challenge").pluck(:taggable_id) + # @challenges = Challenge.where(id: challenge_ids).order("id asc").paginate(page: 1, per_page: 50) end def stories @@ -19,7 +19,7 @@ def stories @page = params[:page].to_i @page = 1 if @page < 1 story_ids = @tag.taggings.where(taggable_type: "Story").pluck(:taggable_id) - @stories = Story.published.where(id: story_ids).order("published_at asc").paginate(page: @page, per_page: 50) + @stories = Story.published.where(id: story_ids).includes(:user).order("published_at asc").paginate(page: @page, per_page: 50) end # # def challenges diff --git a/app/models/tag.rb b/app/models/tag.rb index 2630fe91..5514b6e3 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -9,11 +9,11 @@ def self.calculate_counts end def calculate_counts - story_ids = self.taggings.where(taggable_type: "Story").pluck(:taggable_id) + story_ids = self.taggings.where(taggable_type: "Story").distinct.pluck(:taggable_id) self.stories_count = Story.published.where(id: story_ids).count - challenge_ids = self.taggings.where(taggable_type: "Challenge").pluck(:taggable_id) + challenge_ids = self.taggings.where(taggable_type: "Challenge").distinct.pluck(:taggable_id) self.challenges_count = challenge_ids.length - self.update_columns(stories_count: stories_count, challenges_count: challenges_count) + self.update_columns(stories_count: stories_count, challenges_count: challenges_count) if self.changed? end def stories diff --git a/bin/deploy b/bin/deploy new file mode 100755 index 00000000..9572b73d --- /dev/null +++ b/bin/deploy @@ -0,0 +1,8 @@ +#!/bin/sh +USER=ficly_archive +HOST=export.ficly.com +DIR=export.ficly.com/ # might sometimes be empty! + +rsync -avz --delete tmp/export/ ${USER}@${HOST}:~/${DIR} + +exit 0 diff --git a/db/export.sql.gz b/db/export.sql.gz new file mode 100644 index 00000000..65b57c37 Binary files /dev/null and b/db/export.sql.gz differ diff --git a/lib/tasks/export.rake b/lib/tasks/export.rake index 1faeec70..02c69769 100644 --- a/lib/tasks/export.rake +++ b/lib/tasks/export.rake @@ -82,7 +82,7 @@ namespace :export do task :tags => :environment do api.get("/tags") - total_pages = Tag.where.not(cleaned_tag: [nil, "", '-']).paginate(page: @page, per_page: 250).total_pages + total_pages = Tag.where.not(cleaned_tag: [nil, "", '-']).where("stories_count > 0").paginate(page: @page, per_page: 250).total_pages i = 1 @@ -91,12 +91,18 @@ namespace :export do i += 1 end - Tag.where.not(cleaned_tag: [nil, "", '-']).find_each do |tag| - story_ids = tag.taggings.where(taggable_type: "Story").pluck(:taggable_id) - total_pages = Story.published.where(id: story_ids).order("published_at asc").paginate(page: 1, per_page: 50).total_pages - i = 0 - while i <= total_pages - api.get("/tags/#{tag.cleaned_name}/stories/#{i}") + Tag.where.not(cleaned_tag: [nil, "", '-']).where("stories_count > 0").order("stories_count desc").find_each do |tag| + api.get("/tags/#{tag.cleaned_tag}/stories") + + story_ids = tag.taggings.where(taggable_type: "Story").distinct.pluck(:taggable_id) + total_pages = Story.published.where(id: story_ids).paginate(page: 1, per_page: 50).total_pages + i = 1 + + if total_pages > 1 + while i <= total_pages + api.get("/tags/#{tag.cleaned_tag}/stories/#{i}") + i += 1 + end end end