diff --git a/pluto-models/Gemfile b/pluto-models/Gemfile new file mode 100644 index 0000000..8da954c --- /dev/null +++ b/pluto-models/Gemfile @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' +ruby '>=2.2' +gem 'activerecord' # , '>= 6.1.4' +gem 'activerecord-utils' # , '>= 0.4.0' +gem 'activityutils' # , '>= 0.1.1' +gem 'date-formatter' # , '>= 0.1.1' +gem 'feedfilter' # , '>= 1.1.1' +gem 'feedparser' # , '>= 2.1.2' +gem 'logutils' # , '>= 0.6.1' +gem 'logutils-activerecord' # , '>= 0.2.1' +gem 'props' # , '>= 1.2.0' +gem 'props-activerecord' # , '>= 0.2.0' +gem 'rubocop', require: false +gem 'rubocop-rake', require: false +gem 'textutils' # , '>= 1.4.0' diff --git a/pluto-models/Gemfile.lock b/pluto-models/Gemfile.lock new file mode 100644 index 0000000..a0c1841 --- /dev/null +++ b/pluto-models/Gemfile.lock @@ -0,0 +1,112 @@ +GEM + remote: https://rubygems.org/ + specs: + activemodel (7.1.3.2) + activesupport (= 7.1.3.2) + activerecord (7.1.3.2) + activemodel (= 7.1.3.2) + activesupport (= 7.1.3.2) + timeout (>= 0.4.0) + activerecord-utils (0.4.1) + activerecord + logutils + activesupport (7.1.3.2) + base64 + bigdecimal + concurrent-ruby (~> 1.0, >= 1.0.2) + connection_pool (>= 2.2.5) + drb + i18n (>= 1.6, < 2) + minitest (>= 5.1) + mutex_m + tzinfo (~> 2.0) + activityutils (0.1.2) + logutils (>= 0.6.1) + ast (2.4.2) + base64 (0.2.0) + bigdecimal (3.1.6) + concurrent-ruby (1.2.3) + connection_pool (2.4.1) + date-formatter (0.1.1) + drb (2.2.0) + ruby2_keywords + feedfilter (1.1.1) + textutils (>= 1.0.1) + feedparser (2.2.0) + logutils (>= 0.6.1) + textutils (>= 1.0.0) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + iniparser (1.0.1) + json (2.7.1) + language_server-protocol (3.17.0.3) + logutils (0.6.1) + logutils-activerecord (0.2.1) + activerecord + logutils (>= 0.6.1) + minitest (5.22.2) + mutex_m (0.2.0) + parallel (1.24.0) + parser (3.3.0.5) + ast (~> 2.4.1) + racc + props (1.2.0) + iniparser (>= 0.1.0) + props-activerecord (0.2.0) + activerecord + props (>= 1.2.0) + racc (1.7.3) + rainbow (3.1.1) + regexp_parser (2.9.0) + rexml (3.2.6) + rubocop (1.60.2) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-rake (0.6.0) + rubocop (~> 1.0) + ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) + rubyzip (2.3.2) + textutils (1.4.0) + activesupport + logutils (>= 0.6.1) + props (>= 1.1.2) + rubyzip (>= 1.0.0) + timeout (0.4.1) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.5.0) + +PLATFORMS + x86_64-linux + +DEPENDENCIES + activerecord + activerecord-utils + activityutils + date-formatter + feedfilter + feedparser + logutils + logutils-activerecord + props + props-activerecord + rubocop + rubocop-rake + textutils + +RUBY VERSION + ruby 3.0.2p107 + +BUNDLED WITH + 2.3.5 diff --git a/pluto-models/Rakefile b/pluto-models/Rakefile index 63eeaae..67ec951 100644 --- a/pluto-models/Rakefile +++ b/pluto-models/Rakefile @@ -1,14 +1,15 @@ +# frozen_string_literal: true + require 'hoe' -require './lib/pluto/version.rb' +require './lib/pluto/version' Hoe.spec 'pluto-models' do - self.version = Pluto::VERSION self.summary = "pluto-models - planet schema 'n' models for easy (re)use" self.description = summary - self.urls = ['https://github.com/feedreader/pluto'] + self.urls = { 'home': 'https://github.com/feedreader/pluto' } self.author = 'Gerald Bauer' self.email = 'wwwmake@googlegroups.com' @@ -28,14 +29,12 @@ Hoe.spec 'pluto-models' do ['logutils-activerecord', '>= 0.2.1'], ['props-activerecord', '>= 0.2.0'], ['activityutils', '>= 0.1.1'], - ['activerecord-utils', '>= 0.4.0'], + ['activerecord-utils', '>= 0.4.0'] ] - self.licenses = ['Public Domain'] self.spec_extras = { required_ruby_version: '>= 2.2.2' } - end diff --git a/pluto-models/lib/pluto/config.rb b/pluto-models/lib/pluto/config.rb index 076c322..855027c 100644 --- a/pluto-models/lib/pluto/config.rb +++ b/pluto-models/lib/pluto/config.rb @@ -1,32 +1,29 @@ - - -module Pluto - class Configuration - - def debug=(value) @debug = value; end - def debug?() @debug || false; end - - def logger - ## always return root for now; let's you globally configure e.g. - ## logger.level = :debug etc. - LogUtils::Logger.root - end - end # class Configuration - - - - ## lets you use - ## Pluto.configure do |config| - ## config.debug = true - ## config.logger.level = :debug - ## end - - def self.configure - yield( config ) - end - - def self.config - @config ||= Configuration.new - end - -end # module Pluto +# frozen_string_literal: true + +module Pluto + class Configuration + attr_writer :debug + + def debug? = @debug || false + + def logger + ## always return root for now; let's you globally configure e.g. + ## logger.level = :debug etc. + LogUtils::Logger.root + end + end + + ## lets you use + ## Pluto.configure do |config| + ## config.debug = true + ## config.logger.level = :debug + ## end + + def self.configure + yield(config) + end + + def self.config + @config ||= Configuration.new + end +end diff --git a/pluto-models/lib/pluto/connecter.rb b/pluto-models/lib/pluto/connecter.rb index bf32b8e..7fb0ca8 100644 --- a/pluto-models/lib/pluto/connecter.rb +++ b/pluto-models/lib/pluto/connecter.rb @@ -1,89 +1,74 @@ -# encoding: utf-8 +# frozen_string_literal: true module Pluto + # DB Connecter / Connection Manager + # lets you establish connection + class Connecter + include LogUtils::Logging -# DB Connecter / Connection Manager -# lets you establish connection - -class Connecter - - include LogUtils::Logging - - - - def initialize - # do nothing for now - end - - def debug?() Pluto.config.debug?; end - - - - def connect( config={} ) - - if config.empty? # use/try DATABASE_URL from environment - - logger.debug "ENV['DATBASE_URL'] - >#{ENV['DATABASE_URL']}<" - - db = URI.parse( ENV['DATABASE_URL'] || 'sqlite3:///pluto.db' ) + def initialize + # do nothing for now + end - if db.scheme == 'postgres' - config = { - adapter: 'postgresql', - host: db.host, - port: db.port, - username: db.user, - password: db.password, - database: db.path[1..-1], - encoding: 'utf8' - } - else # assume sqlite3 - config = { - adapter: db.scheme, # sqlite3 - database: db.path[1..-1] # pluto.db (NB: cut off leading /, thus 1..-1) - } + def debug? = Pluto.config.debug? + + def connect(config = {}) + if config.empty? # use/try DATABASE_URL from environment + + logger.debug "ENV['DATBASE_URL'] - >#{ENV['DATABASE_URL']}<" + + db = URI.parse(ENV['DATABASE_URL'] || 'sqlite3:///pluto.db') + + config = if db.scheme == 'postgres' + { + adapter: 'postgresql', + host: db.host, + port: db.port, + username: db.user, + password: db.password, + database: db.path[1..], + encoding: 'utf8' + } + else # assume sqlite3 + { + adapter: db.scheme, # sqlite3 + database: db.path[1..] # pluto.db (NB: cut off leading /, thus 1..-1) + } + end end - end # if config.nil? - logger.info 'db settings:' - logger.info config.pretty_inspect + logger.info 'db settings:' + logger.info config.pretty_inspect - ### for dbbrowser and other tools add to ActiveRecord + ### for dbbrowser and other tools add to ActiveRecord - if ActiveRecord::Base.configurations.nil? # todo/check: can this ever happen? remove? - logger.debug "ActiveRecord configurations nil - set to empty hash" - ActiveRecord::Base.configurations = {} # make it an empty hash - end + if ActiveRecord::Base.configurations.nil? # todo/check: can this ever happen? remove? + logger.debug 'ActiveRecord configurations nil - set to empty hash' + ActiveRecord::Base.configurations = {} # make it an empty hash + end - ## todo/fix: remove debug? option - why? why not? - ## (just) use logger level eg. logger.debug - if debug? - logger.debug 'ar configurations (before):' - logger.debug ActiveRecord::Base.configurations.pretty_inspect - end + ## todo/fix: remove debug? option - why? why not? + ## (just) use logger level eg. logger.debug + if debug? + logger.debug 'ar configurations (before):' + logger.debug ActiveRecord::Base.configurations.pretty_inspect + end - # note: for now always use pluto key for config storage - configs = ActiveRecord::Base.configurations.to_h.reject { |db_config| db_config.env_name == 'pluto' } - configs['pluto'] = config + configurations = ActiveRecord::Base.configurations.configs_for(env_name: 'pluto') + configurations << config - ActiveRecord::Base.configurations = configs + ActiveRecord::Base.configurations = configurations - if debug? - logger.debug 'ar configurations (after):' - logger.debug ActiveRecord::Base.configurations.pretty_inspect - end + if debug? + logger.debug 'ar configurations (after):' + logger.debug ActiveRecord::Base.configurations.pretty_inspect + end + # for debugging - disable for production use + ActiveRecord::Base.logger = Logger.new($stdout) if debug? - # for debugging - disable for production use - if debug? - ActiveRecord::Base.logger = Logger.new( STDOUT ) + ActiveRecord::Base.establish_connection(config) end - - ActiveRecord::Base.establish_connection( config ) - end # method connect - - -end # class Connecter - -end # module Pluto + end +end diff --git a/pluto-models/lib/pluto/models.rb b/pluto-models/lib/pluto/models.rb index 1c02bb8..46c04b2 100644 --- a/pluto-models/lib/pluto/models.rb +++ b/pluto-models/lib/pluto/models.rb @@ -1,41 +1,37 @@ -# encoding: utf-8 +# frozen_string_literal: true # core and stdlibs require 'yaml' require 'json' require 'uri' -require 'pp' require 'fileutils' require 'date' require 'time' require 'digest/md5' -require 'logger' # Note: use for ActiveRecord::Base.logger = Logger.new( STDOUT ) for now - +require 'logger' # NOTE: use for ActiveRecord::Base.logger = Logger.new( STDOUT ) for now # 3rd party ruby gems/libs require 'active_record' -require 'props' # manage settings/env +require 'props' # manage settings/env require 'logutils' require 'textutils' require 'feedparser' require 'feedfilter' require 'date/formatter' - ## add more activerecords addons/utils -require 'activerecord/utils' # add macros e.g. read_attr_w_fallbacks etc. +require 'activerecord/utils' # add macros e.g. read_attr_w_fallbacks etc. require 'activityutils' require 'props/activerecord' require 'logutils/activerecord' - # our own code -require 'pluto/version' # note: let version always get first +require 'pluto/version' # NOTE: let version always get first require 'pluto/config' require 'pluto/schema' @@ -48,22 +44,19 @@ require 'pluto/connecter' - module Pluto - def self.create CreateDb.new.up - ConfDb::Model::Prop.create!( key: 'db.schema.planet.version', value: VERSION ) + ConfDb::Model::Prop.create!(key: 'db.schema.planet.version', value: VERSION) end def self.create_all LogDb.create # add logs table ConfDb.create # add props table - ActivityDb::CreateDb.new.up # todo/check - use ActivityDb.create if exists??? + ActivityDb::CreateDb.new.up # todo/check - use ActivityDb.create if exists??? Pluto.create end - def self.auto_migrate! # first time? - auto-run db migratation, that is, create db tables unless LogDb::Model::Log.table_exists? @@ -76,54 +69,51 @@ def self.auto_migrate! ## fix: change to Model from Models unless ActivityDb::Models::Activity.table_exists? - ActivityDb::CreateDb.new.up # todo/check - use ActivityDb.create if exists??? - end - - unless Model::Feed.table_exists? - Pluto.create + ActivityDb::CreateDb.new.up # todo/check - use ActivityDb.create if exists??? end - end # method auto_migrate! + return if Model::Feed.table_exists? + Pluto.create + end - def self.connect( config={} ) # convenience shortcut without (w/o) automigrate - Connecter.new.connect( config ) + # convenience shortcut without (w/o) automigrate + def self.connect(config = {}) + Connecter.new.connect(config) end - def self.connect!( config={} ) # convenience shortcut w/ automigrate - Pluto.connect( config ) + # convenience shortcut w/ automigrate + def self.connect!(config = {}) + Pluto.connect(config) Pluto.auto_migrate! end - def self.setup_in_memory_db # Database Setup & Config - ActiveRecord::Base.logger = Logger.new( STDOUT ) + ActiveRecord::Base.logger = Logger.new($stdout) ## ActiveRecord::Base.colorize_logging = false - no longer exists - check new api/config setting? - Pluto.connect( adapter: 'sqlite3', - database: ':memory:' ) + Pluto.connect(adapter: 'sqlite3', + database: ':memory:') ## build schema Pluto.create_all - end # setup_in_memory_dd - + end ######################################### ## let's put test configuration in its own namespace / module - class Test ## todo/check: works with module too? use a module - why? why not? - + ## todo/check: works with module too? use a module - why? why not? + class Test #### # todo/fix: find a better way to configure shared test datasets - why? why not? # note: use one-up (..) directory for now as default - why? why not? - def self.data_dir() @data_dir ||= '../test'; end - def self.data_dir=( path ) @data_dir = path; end - end # class Test - -end # module Pluto - - + def self.data_dir = @data_dir ||= '../test' + class << self + attr_writer :data_dir + end + end +end # say hello -puts Pluto.banner if $DEBUG || (defined?($RUBYLIBS_DEBUG) && $RUBYLIBS_DEBUG) +puts Pluto.banner if $DEBUG || (defined?($RUBYLIBS_DEBUG) && $RUBYLIBS_DEBUG) diff --git a/pluto-models/lib/pluto/models/feed.rb b/pluto-models/lib/pluto/models/feed.rb index 26d70f2..98f8dec 100644 --- a/pluto-models/lib/pluto/models/feed.rb +++ b/pluto-models/lib/pluto/models/feed.rb @@ -1,327 +1,305 @@ -# encoding: utf-8 - +# frozen_string_literal: true module Pluto module Model + class Feed < ActiveRecord::Base + ## logging w/ ActiveRecord + ## todo/check: check if logger instance method is present by default? + ## only class method present? + ## what's the best way to add logging to activerecord (use "builtin" machinery??) + def debug? = Pluto.config.debug? -class Feed < ActiveRecord::Base - -## logging w/ ActiveRecord -## todo/check: check if logger instance method is present by default? -## only class method present? -## what's the best way to add logging to activerecord (use "builtin" machinery??) - + self.table_name = 'feeds' - def debug?() Pluto.config.debug?; end + has_many :items + has_many :subscriptions + has_many :sites, through: :subscriptions + ## todo/fix: + ## use a module ref or something; do NOT include all methods - why? why not? + include TextUtils::HypertextHelper ## e.g. lets us use strip_tags( ht ) + def self.latest + # NOTE: order by first non-null datetime field + # coalesce - supported by sqlite (yes), postgres (yes) - self.table_name = 'feeds' + # NOTE: if not updated or published use hardcoded 1970-01-01 for now + ## was: order( "coalesce(updated,published,'1970-01-01') desc" ) + order(Arel.sql("coalesce(feeds.items_last_updated,'1970-01-01') desc")) + end - has_many :items - has_many :subscriptions - has_many :sites, :through => :subscriptions + ################################## + # attribute reader aliases + # + # note: CANNOT use alias_method :name, :title + # will NOT work for non-existing/on-demand-generated methods in activerecord + # + # use rails alias_attribute :new, :old (incl. reader/predicate/writer) + # or use alias_attr, alias_attr_reader, alias_attr_writer from activerecord/utils + + alias_attr_reader :name, :title # alias for title + alias_attr_reader :description, :summary # alias for summary + alias_attr_reader :desc, :summary # alias(2) for summary + alias_attr_reader :subtitle, :summary # alias(3) for summary + alias_attr_reader :link, :url # alias for url + alias_attr_reader :html_url, :url # alias(2) for url + alias_attr_reader :feed, :feed_url # alias for feed_url + + alias_attr_reader :author_name, :author # alias for author + alias_attr_reader :owner_name, :author # alias(2) for author + alias_attr_reader :owner, :author # alias(3) for author + alias_attr_reader :author_email, :email # alias for email + alias_attr_reader :author_email, :email # alias(2) for email + + ################# + ## attributes with fallbacks or (auto-)backups - use feed.data. for "raw" / "original" access + def url = read_attribute_w_fallbacks(:url, :auto_url) + def title = read_attribute_w_fallbacks(:title, :auto_title) + def feed_url = read_attribute_w_fallbacks(:feed_url, :auto_feed_url) + + def url? = url.present? + def title? = title.present? + def feed_url? = feed_url.present? + + ## note: + ## only use fallback for updated, that is, updated (or published) + ## ~~do NOT use fallback for published / created -- why? why not?~~ + ## add items_last_updated to updated as last fall back - why? why not? + def updated = read_attribute_w_fallbacks(:updated, :published) + def published = read_attribute_w_fallbacks(:published, :updated) + + def updated? = updated.present? + def published? = published.present? + + ############# + # add convenience date attribute helpers / readers + # - what to return if date is nil? - return nil or empty string or 'n/a' or '?' - why? why not? + # + # date + # date_iso | date_iso8601 + # date_822 | date_rfc2822 | date_rfc822 + def date = updated - ## todo/fix: - ## use a module ref or something; do NOT include all methods - why? why not? - include TextUtils::HypertextHelper ## e.g. lets us use strip_tags( ht ) + def date_iso = date ? date.iso8601 : '' + alias date_iso8601 date_iso + def date_822 = date ? date.rfc822 : '' + alias date_rfc2822 date_822 + alias date_rfc822 date_822 - def self.latest - # note: order by first non-null datetime field - # coalesce - supported by sqlite (yes), postgres (yes) + ## "raw" access via data "proxy" helper + ## e.g. use feed.data.updated + ## feed.data.updated? etc. + class Data + def initialize(feed) = @feed = feed - # note: if not updated or published use hardcoded 1970-01-01 for now - ## was: order( "coalesce(updated,published,'1970-01-01') desc" ) - order( Arel.sql( "coalesce(feeds.items_last_updated,'1970-01-01') desc" ) ) - end + def url + @feed.read_attribute(:url) + end - ################################## - # attribute reader aliases - # - # note: CANNOT use alias_method :name, :title - # will NOT work for non-existing/on-demand-generated methods in activerecord - # - # use rails alias_attribute :new, :old (incl. reader/predicate/writer) - # or use alias_attr, alias_attr_reader, alias_attr_writer from activerecord/utils - - alias_attr_reader :name, :title # alias for title - alias_attr_reader :description, :summary # alias for summary - alias_attr_reader :desc, :summary # alias(2) for summary - alias_attr_reader :subtitle, :summary # alias(3) for summary - alias_attr_reader :link, :url # alias for url - alias_attr_reader :html_url, :url # alias(2) for url - alias_attr_reader :feed, :feed_url # alias for feed_url - - alias_attr_reader :author_name, :author # alias for author - alias_attr_reader :owner_name, :author # alias(2) for author - alias_attr_reader :owner, :author # alias(3) for author - alias_attr_reader :author_email, :email # alias for email - alias_attr_reader :author_email, :email # alias(2) for email - - - ################# - ## attributes with fallbacks or (auto-)backups - use feed.data. for "raw" / "original" access - def url() read_attribute_w_fallbacks( :url, :auto_url ); end - def title() read_attribute_w_fallbacks( :title, :auto_title ); end - def feed_url() read_attribute_w_fallbacks( :feed_url, :auto_feed_url ); end - - def url?() url.present?; end - def title?() title.present?; end - def feed_url?() feed_url.present?; end - - ## note: - ## only use fallback for updated, that is, updated (or published) - ## ~~do NOT use fallback for published / created -- why? why not?~~ - ## add items_last_updated to updated as last fall back - why? why not? - def updated() read_attribute_w_fallbacks( :updated, :published ); end - def published() read_attribute_w_fallbacks( :published, :updated, ); end - - def updated?() updated.present?; end - def published?() published.present?; end - - ############# - # add convenience date attribute helpers / readers - # - what to return if date is nil? - return nil or empty string or 'n/a' or '?' - why? why not? - # - # date - # date_iso | date_iso8601 - # date_822 | date_rfc2822 | date_rfc822 - - def date() updated; end - - def date_iso() date ? date.iso8601 : ''; end - alias_method :date_iso8601, :date_iso - - def date_822() date ? date.rfc822 : ''; end - alias_method :date_rfc2822, :date_822 - alias_method :date_rfc822, :date_822 - - - ## "raw" access via data "proxy" helper - ## e.g. use feed.data.updated - ## feed.data.updated? etc. - class Data - def initialize( feed ) @feed = feed; end - - def url() @feed.read_attribute( :url ); end # "regular" url incl. auto_url fallback / (auto-)backup - def title() @feed.read_attribute( :title ); end - def feed_url() @feed.read_attribute( :feed_url ); end - def url?() url.present?; end - def title?() title.present?; end - def feed_url?() feed_url.present?; end - - def updated() @feed.read_attribute(:updated); end # "regular" updated incl. published fallback - def published() @feed.read_attribute(:published); end # "regular" published incl. updated fallback - def updated?() updated.present?; end - def published?() published.present?; end - end # class Data - ## use a different name for data - why? why not? - ## e.g. inner, internal, readonly or r, raw, table, direct, or ??? - def data() @data ||= Data.new( self ); end - - - def deep_update_from_struct!( data ) - - logger = LogUtils::Logger.root - - ## note: handle case with empty feed, that is, feed with NO items / entries - ## (e.g. data.items.size == 0). - if data.items.size > 0 - - ##### - ## apply some fix-up for "broken" feed data - fix_dates( data ) - - - ###### - ## check for filters (includes/excludes) if present - ## for now just check for includes - ## - if includes.present? - includesFilter = FeedFilter::IncludeFilters.new( includes ) - else - includesFilter = nil + def title = @feed.read_attribute(:title) + def feed_url = @feed.read_attribute(:feed_url) + def url? = url.present? + def title? = title.present? + def feed_url? = feed_url.present? + + # "regular" updated incl. published fallback + def updated = @feed.read_attribute(:updated) + # "regular" published incl. updated fallback + def published = @feed.read_attribute(:published) + def updated? = updated.present? + def published? = published.present? end - data.items.each do |item| - if includesFilter && includesFilter.match_item?( item ) == false - logger.info "** SKIPPING | #{item.title}" - logger.info " no include terms match: #{includes}" - next ## skip to next item - end + ## use a different name for data - why? why not? + ## e.g. inner, internal, readonly or r, raw, table, direct, or ??? + def data = @data ||= Data.new(self) + + def deep_update_from_struct!(data) + logger = LogUtils::Logger.root + + ## note: handle case with empty feed, that is, feed with NO items / entries + ## (e.g. data.items.size == 0). + if data.items.size.positive? + + ##### + ## apply some fix-up for "broken" feed data + fix_dates(data) + + ###### + ## check for filters (includes/excludes) if present + ## for now just check for includes + ## + includes_filter = (FeedFilter::IncludeFilters.new(includes) if includes.present?) + + data.items.each do |item| + if includes_filter && includes_filter.match_item?(item) == false + logger.info "** SKIPPING | #{item.title}" + logger.info " no include terms match: #{includes}" + next ## skip to next item + end + + item_rec = Item.find_by_guid(item.guid) + if item_rec.nil? + item_rec = Item.new + logger.info "** NEW | #{item.title}" + else + ## todo: check if any attribs changed + logger.info "UPDATE | #{item.title}" + end + + item_rec.feed_id = id # feed_rec.id - add feed_id fk_ref + item_rec.fetched = fetched # feed_rec.fetched + + item_rec.update_from_struct!(item) + end - item_rec = Item.find_by_guid( item.guid ) - if item_rec.nil? - item_rec = Item.new - logger.info "** NEW | #{item.title}" - else - ## todo: check if any attribs changed - logger.info "UPDATE | #{item.title}" - end + ### + # delete (old) feed items if no longer in feed AND + # date range is in (lastest/current) feed list + # + # thanks to Harry Wood + # see https://github.com/feedreader/pluto/pull/16 + # for more comments + + # todo/fix: use a delete feature/command line flag to make it optional - why? why not? + + guids_in_feed = data.items.map(&:guid) + earliest_still_in_feed = data.items.min_by(&:published).published + + items_no_longer_present = + Item + .where(feed_id: id) + .where.not(published: nil) + .where('published > ?', earliest_still_in_feed) + .where.not(guid: guids_in_feed) + + unless items_no_longer_present.empty? + logger.info "#{items_no_longer_present.size} items no longer present in the feed (presumed removed at source). Deleting from planet db" + items_no_longer_present.each do |item| + logger.info "** DELETE | #{item.title}" + item.destroy + end + end - item_rec.feed_id = id # feed_rec.id - add feed_id fk_ref - item_rec.fetched = fetched # feed_rec.fetched + # update cached value last published for item + ## todo/check: force reload of items - why? why not?? + last_item_rec = items.latest.limit(1).first # NOTE: limit(1) will return relation/arrar - use first to get first element or nil from ary + if last_item_rec&.updated? ## note: checks for updated & published with attr_reader_w_fallback + self.items_last_updated = last_item_rec.updated + ## save! ## note: will get save w/ update_from_struct! - why? why not?? + # else + ## skip - no updated / published present + end + end - item_rec.update_from_struct!( item ) - end # each item + update_from_struct!(data) + end + # try to get date from slug in url + # e.g. /news/2019-10-17-growing-ruby-together + FIX_DATE_SLUG_RE = /\b + (?[0-9]{4}) + - + (?[0-9]{2}) + - + (?[0-9]{2}) + \b/x + + ################################################### + # helpers to fix-up some "broken" feed data + def fix_dates(data) + ## check for missing / no dates + ## examples + ## - rubytogether feed @ https://rubytogether.org/news.xml + data.items.each do |item| + ## try to get date from slug in url + ## e.g. /news/2019-10-17-growing-ruby-together + next unless item.updated.nil? && + item.published.nil? && (m = FIX_DATE_SLUG_RE.match(item.url)) + + ## todo/fix: make sure DateTime gets utc (no timezone/offset +000) + published = DateTime.new(m[:year].to_i(10), + m[:month].to_i(10), + m[:day].to_i(10)) + item.published_local = published + item.published = published + end - ### - # delete (old) feed items if no longer in feed AND - # date range is in (lastest/current) feed list - # - # thanks to Harry Wood - # see https://github.com/feedreader/pluto/pull/16 - # for more comments - - # todo/fix: use a delete feature/command line flag to make it optional - why? why not? - - guids_in_feed = data.items.map {|item| item.guid } - earliest_still_in_feed = data.items.min_by {|item| item.published }.published - - items_no_longer_present = - Item - .where(feed_id: id) - .where.not(published: nil) - .where("published > ?", earliest_still_in_feed) - .where.not(guid: guids_in_feed) - - unless items_no_longer_present.empty? - logger.info "#{items_no_longer_present.size} items no longer present in the feed (presumed removed at source). Deleting from planet db" - items_no_longer_present.each do |item| - logger.info "** DELETE | #{item.title}" - item.destroy + ## check if all updated dates are the same (uniq count is 1) + ## AND if all published dates are present + ## than assume "fake" updated dates and nullify updated dates + ## example real-world "messed-up" feeds include: + ## - https://bundler.io/blog/feed.xml + ## - https://dry-rb.org/feed.xml + ## + ## todo/check - limit to atom feed format only - why? why not? + + count = data.items.size + count_published = data.items.reduce(0) do |cnt, item| + cnt += 1 if item.published + cnt end - end + return unless count == count_published + + uniq_count_updated = 0 + last_updated = nil - # update cached value last published for item - ## todo/check: force reload of items - why? why not?? - last_item_rec = items.latest.limit(1).first # note limit(1) will return relation/arrar - use first to get first element or nil from ary - if last_item_rec - if last_item_rec.updated? ## note: checks for updated & published with attr_reader_w_fallback - self.items_last_updated = last_item_rec.updated - ## save! ## note: will get save w/ update_from_struct! - why? why not?? - else - ## skip - no updated / published present + data.items.each do |item| + uniq_count_updated += 1 if item.updated != last_updated + last_updated = item.updated end - end - end # check for if data.items.size > 0 (that is, feed has feed items/entries) - - update_from_struct!( data ) - end # method deep_update_from_struct! - - - # try to get date from slug in url - # e.g. /news/2019-10-17-growing-ruby-together - FIX_DATE_SLUG_RE = %r{\b - (?[0-9]{4}) - - - (?[0-9]{2}) - - - (?[0-9]{2}) - \b}x - - ################################################### - # helpers to fix-up some "broken" feed data - def fix_dates( data ) - - ## check for missing / no dates - ## examples - ## - rubytogether feed @ https://rubytogether.org/news.xml - data.items.each do |item| - if item.updated.nil? && - item.published.nil? - ## try to get date from slug in url - ## e.g. /news/2019-10-17-growing-ruby-together - if (m=FIX_DATE_SLUG_RE.match( item.url )) - ## todo/fix: make sure DateTime gets utc (no timezone/offset +000) - published = DateTime.new( m[:year].to_i(10), - m[:month].to_i(10), - m[:day].to_i(10) ) - item.published_local = published - item.published = published - end - end - end - - ## check if all updated dates are the same (uniq count is 1) - ## AND if all published dates are present - ## than assume "fake" updated dates and nullify updated dates - ## example real-world "messed-up" feeds include: - ## - https://bundler.io/blog/feed.xml - ## - https://dry-rb.org/feed.xml - ## - ## todo/check - limit to atom feed format only - why? why not? - - count = data.items.size - count_published = data.items.reduce( 0 ) {|count,item| count += 1 if item.published; count } - - if count == count_published - uniq_count_updated = 0 - last_updated = nil - - data.items.each do |item| - uniq_count_updated += 1 if item.updated != last_updated - last_updated = item.updated - end + return unless uniq_count_updated == 1 - if uniq_count_updated == 1 - puts "bingo!! nullify all updated dates" + puts 'bingo!! nullify all updated dates' ## todo/fix: log report updated date fix!!!! data.items.each do |item| item.updated = nil item.updated_local = nil end end - end - end - - def update_from_struct!( data ) - logger = LogUtils::Logger.root - -## -# todo: -## strip all tags from summary (subtitle) -## limit to 255 chars -## e.g. summary (subtitle) such as this exist -## This is a low-traffic announce-only list for people interested -## in hearing news about Polymer (http://polymer-project.org). -## The higher-traffic mailing list for all kinds of discussion is -## https://groups.google.com/group/polymer-dev - - feed_attribs = { - format: data.format, - updated: data.updated, - published: data.published, - summary: data.summary, - generator: data.generator.to_s, ## note: use single-line/string generator stringified -- might return null (if no data) - - ## note: always auto-update auto_* fields for now - auto_title: data.title, - auto_url: data.url, - auto_feed_url: data.feed_url, - } - - if debug? - ## puts "*** dump feed_attribs:" - ## pp feed_attribs - logger.debug "*** dump feed_attribs w/ class types:" - feed_attribs.each do |key,value| - logger.debug " #{key}: >#{value}< : #{value.class.name}" + def update_from_struct!(data) + logger = LogUtils::Logger.root + + ## + # todo: + ## strip all tags from summary (subtitle) + ## limit to 255 chars + ## e.g. summary (subtitle) such as this exist + ## This is a low-traffic announce-only list for people interested + ## in hearing news about Polymer (http://polymer-project.org). + ## The higher-traffic mailing list for all kinds of discussion is + ## https://groups.google.com/group/polymer-dev + + feed_attribs = { + format: data.format, + updated: data.updated, + published: data.published, + summary: data.summary, + generator: data.generator.to_s, ## note: use single-line/string generator stringified -- might return null (if no data) + + ## note: always auto-update auto_* fields for now + auto_title: data.title, + auto_url: data.url, + auto_feed_url: data.feed_url + } + + if debug? + ## puts "*** dump feed_attribs:" + ## pp feed_attribs + logger.debug '*** dump feed_attribs w/ class types:' + feed_attribs.each do |key, value| + logger.debug " #{key}: >#{value}< : #{value.class.name}" + end end - end - update!( feed_attribs ) + update!(feed_attribs) + end + end end - -end # class Feed - - - end # module Model -end # module Pluto +end diff --git a/pluto-models/lib/pluto/models/forward.rb b/pluto-models/lib/pluto/models/forward.rb index 5dd9920..6f8f6f0 100644 --- a/pluto-models/lib/pluto/models/forward.rb +++ b/pluto-models/lib/pluto/models/forward.rb @@ -1,19 +1,16 @@ -# encoding: utf-8 +# frozen_string_literal: true module Pluto module Model + ####### + # ActivityDB -####### -# ActivityDB + # add shortcut/alias + ## fix: use Model instead of Models + Activity = ActivityDb::Models::Activity + end -# add shortcut/alias -## fix: use Model instead of Models -Activity = ActivityDb::Models::Activity - - end # module Model - - # note: convenience alias for Model + # NOTE: convenience alias for Model # lets you use include Pluto::Models Models = Model - -end # module Pluto +end diff --git a/pluto-models/lib/pluto/models/item.rb b/pluto-models/lib/pluto/models/item.rb index 865872a..9958d34 100644 --- a/pluto-models/lib/pluto/models/item.rb +++ b/pluto-models/lib/pluto/models/item.rb @@ -1,122 +1,113 @@ -# encoding: utf-8 +# frozen_string_literal: true module Pluto module Model + class Item < ActiveRecord::Base + ## logging w/ ActiveRecord + ## todo/check: check if logger instance method is present by default? + ## only class method present? + ## what's the best way to add logging to activerecord (use "builtin" machinery??) -class Item < ActiveRecord::Base + def debug? = Pluto.config.debug? -## logging w/ ActiveRecord -## todo/check: check if logger instance method is present by default? -## only class method present? -## what's the best way to add logging to activerecord (use "builtin" machinery??) + self.table_name = 'items' + belongs_to :feed - def debug?() Pluto.config.debug?; end + ## todo/fix: + ## use a module ref or something; do NOT include all methods - why? why not? + include TextUtils::HypertextHelper ## e.g. lets us use strip_tags( ht ) + include FeedFilter::AdsFilter ## e.g. lets us use strip_ads( ht ) + ################################## + # attribute reader aliases + alias_attr_reader :name, :title # alias for title + alias_attr_reader :description, :summary # alias for summary -- also add descr shortcut?? + alias_attr_reader :desc, :summary # alias (2) for summary -- also add descr shortcut?? + alias_attr_reader :link, :url # alias for url - self.table_name = 'items' + def self.latest + # NOTE: order by first non-null datetime field + # coalesce - supported by sqlite (yes), postgres (yes) - belongs_to :feed - - ## todo/fix: - ## use a module ref or something; do NOT include all methods - why? why not? - include TextUtils::HypertextHelper ## e.g. lets us use strip_tags( ht ) - include FeedFilter::AdsFilter ## e.g. lets us use strip_ads( ht ) - - - ################################## - # attribute reader aliases - alias_attr_reader :name, :title # alias for title - alias_attr_reader :description, :summary # alias for summary -- also add descr shortcut?? - alias_attr_reader :desc, :summary # alias (2) for summary -- also add descr shortcut?? - alias_attr_reader :link, :url # alias for url - - - def self.latest - # note: order by first non-null datetime field - # coalesce - supported by sqlite (yes), postgres (yes) - - # note: if not updated,published use hardcoded 1970-01-01 for now - order( Arel.sql( "coalesce(items.updated,items.published,'1970-01-01') desc" ) ) - end + # NOTE: if not updated,published use hardcoded 1970-01-01 for now + order(Arel.sql("coalesce(items.updated,items.published,'1970-01-01') desc")) + end - ## note: - ## only use fallback for updated, that is, updated (or published) - ## ~~do NOT use fallback for published / created -- why? why not?~~ - def updated() read_attribute_w_fallbacks( :updated, :published ); end - def published() read_attribute_w_fallbacks( :published, :updated ); end + ## note: + ## only use fallback for updated, that is, updated (or published) + ## ~~do NOT use fallback for published / created -- why? why not?~~ + def updated = read_attribute_w_fallbacks(:updated, :published) + def published = read_attribute_w_fallbacks(:published, :updated) - def updated?() updated.present?; end - def published?() published.present?; end + def updated? = updated.present? + def published? = published.present? - ############# - # add convenience date attribute helpers / readers - # - what to return if date is nil? - return nil or empty string or 'n/a' or '?' - why? why not? - # - # date - # date_iso | date_iso8601 - # date_822 | date_rfc2822 | date_rfc822 + ############# + # add convenience date attribute helpers / readers + # - what to return if date is nil? - return nil or empty string or 'n/a' or '?' - why? why not? + # + # date + # date_iso | date_iso8601 + # date_822 | date_rfc2822 | date_rfc822 - def date() updated; end + def date = updated - def date_iso() date ? date.iso8601 : ''; end - alias_method :date_iso8601, :date_iso + def date_iso = date ? date.iso8601 : '' + alias date_iso8601 date_iso - def date_822() date ? date.rfc822 : ''; end - alias_method :date_rfc2822, :date_822 - alias_method :date_rfc822, :date_822 + def date_822 = date ? date.rfc822 : '' + alias date_rfc2822 date_822 + alias date_rfc822 date_822 + ## "raw" access via data "proxy" helper + ## e.g. use item.data.updated + ## item.data.updated? etc. + class Data + def initialize(item) = @item = item - ## "raw" access via data "proxy" helper - ## e.g. use item.data.updated - ## item.data.updated? etc. - class Data - def initialize( item ) @item = item; end + # "regular" updated incl. published fallback + def updated = @item.read_attribute(:updated) + def published = @item.read_attribute(:published) - def updated() @item.read_attribute(:updated); end # "regular" updated incl. published fallback - def published() @item.read_attribute(:published); end + def updated? = updated.present? + def published? = published.present? + end - def updated?() updated.present?; end - def published?() published.present?; end - end # class Data - ## use a different name for data - why? why not? - ## e.g. inner, internal, readonly or r, raw, table, direct, or ??? - def data() @data ||= Data.new( self ); end + ## use a different name for data - why? why not? + ## e.g. inner, internal, readonly or r, raw, table, direct, or ??? + def data = @data ||= Data.new(self) + def update_from_struct!(data) + logger = LogUtils::Logger.root - def update_from_struct!( data ) + ## check: new item/record? not saved? add guid + # otherwise do not add guid - why? why not? - logger = LogUtils::Logger.root + ## note: for now also strip ads in summary + ## fix/todo: summary (in the future) is supposed to be only plain vanilla text - ## check: new item/record? not saved? add guid - # otherwise do not add guid - why? why not? + item_attribs = { + guid: data.guid, # TODO: only add for new records??? + title: data.title ? strip_tags(data.title)[0...255] : data.title, ## limit to 255 chars; strip tags + url: data.url, + summary: data.summary.blank? ? data.summary : strip_ads(data.summary).strip, + content: data.content.blank? ? data.content : strip_ads(data.content).strip, + updated: data.updated, + published: data.published + } - ## note: for now also strip ads in summary - ## fix/todo: summary (in the future) is supposed to be only plain vanilla text + if debug? + logger.debug '*** dump item_attribs w/ class types:' + item_attribs.each do |key, value| + next if %i[summary content].include?(key) # skip summary n content - item_attribs = { - guid: data.guid, # todo: only add for new records??? - title: data.title ? strip_tags(data.title)[0...255] : data.title, ## limit to 255 chars; strip tags - url: data.url, - summary: data.summary.blank? ? data.summary : strip_ads( data.summary ).strip, - content: data.content.blank? ? data.content : strip_ads( data.content ).strip, - updated: data.updated, - published: data.published, - } + logger.debug " #{key}: >#{value}< : #{value.class.name}" + end + end - if debug? - logger.debug "*** dump item_attribs w/ class types:" - item_attribs.each do |key,value| - next if [:summary,:content].include?( key ) # skip summary n content - logger.debug " #{key}: >#{value}< : #{value.class.name}" + update!(item_attribs) end end - - update!( item_attribs ) end - -end # class Item - - - end # module Model -end # module Pluto +end diff --git a/pluto-models/lib/pluto/models/site.rb b/pluto-models/lib/pluto/models/site.rb index 7074e32..5c90ccd 100644 --- a/pluto-models/lib/pluto/models/site.rb +++ b/pluto-models/lib/pluto/models/site.rb @@ -1,203 +1,202 @@ -# encoding: utf-8 +# frozen_string_literal: true module Pluto module Model - -class Site < ActiveRecord::Base - - - - self.table_name = 'sites' - - has_many :subscriptions - has_many :feeds, :through => :subscriptions - has_many :items, :through => :feeds - - - ################################## - # attribute reader aliases - alias_attr_reader :name, :title # alias for title - - alias_attr_reader :owner_name, :author # alias for author - alias_attr_reader :owner, :author # alias(2) for author - alias_attr_reader :author_name, :author # alias(3) for author - - alias_attr_reader :owner_email, :email # alias for email - alias_attr_reader :author_email, :email # alias(2) for email - - - - def self.deep_create_or_update_from_hash!( name, config, opts={} ) - - ## note: allow (optional) config of site key too - site_key = config['key'] || config['slug'] - if site_key.nil? - ## if no key configured; use (file)name; remove -_ chars - ## e.g. jekyll-meta becomes jekyllmeta etc. - site_key = name.downcase.gsub( /[\-_]/, '' ) - end - - site_rec = Site.find_by_key( site_key ) - if site_rec.nil? - site_rec = Site.new - site_rec.key = site_key - end - - site_rec.deep_update_from_hash!( config, opts ) - site_rec - end - - - def deep_update_from_hash!( config, opts={} ) - - logger = LogUtils::Logger.root - - site_attribs = { - title: config['title'] || config['name'], # support either title or name - url: config['source'] || config['url'], # support source or url for source url for auto-update (optional) - author: config['author'] || config['owner'], - email: config['email'], - updated: Time.now, ## track last_update via pluto (w/ update_subscription_for fn) - } - - ## note: allow (optional) config of site key too - site_key = config['key'] || config['slug'] - site_attribs[:key] = site_key if site_key - - - logger.debug "site_attribs: #{site_attribs.inspect}" - - if new_record? - ## use object_id: site.id and object_type: Site - ## change - model/table/schema!!! - Activity.create!( text: "new site >#{key}< - #{title}" ) - end - - update!( site_attribs ) - - - # -- log update activity - ## Activity.create!( text: "update subscriptions for site >#{key}<" ) - - #### todo/fix: - ## double check - how to handle delete - ## feeds might get referenced by other sites - ## cannot just delete feeds; only save to delete join table (subscriptions) - ## check if feed "lingers" on with no reference (to site)??? - - # clean out subscriptions and add again - logger.debug "before site.subscriptions.delete_all - count: #{subscriptions.count}" - # note: use destroy_all NOT delete_all (delete_all tries by default only nullify) - subscriptions.destroy_all - logger.debug "after site.subscriptions.delete_all - count: #{subscriptions.count}" - - - config.each do |k, v| - - ## todo: downcase key - why ??? why not??? - - # skip "top-level" feed keys e.g. title, etc. or planet planet sections (e.g. planet,defaults) - next if ['key','slug', - 'title','name','name2','title2','subtitle', - 'source', 'url', - 'include','includes','exclude','excludes', - 'feeds', - 'author', 'owner', 'email', - 'planet','defaults'].include?( k ) - - ### todo/check: - ## check value - must be hash - # check if url or feed_url present - # that is, check for required props/key-value pairs - - feed_key = k.to_s.dup - feed_hash = v - - # todo/fix: use title from feed? - # e.g. fill up auto_title, auto_url, etc. - - feed_attribs = { - feed_url: feed_hash[ 'feed' ] || feed_hash[ 'feed_url' ] || feed_hash[ 'xml_url' ], - url: feed_hash[ 'link' ] || feed_hash[ 'url' ] || feed_hash[ 'html_url' ], - title: feed_hash[ 'title' ] || feed_hash[ 'name' ], -## note: title2 no longer supported; use summary or subtitle? -### title2: feed_hash[ 'title2' ] || feed_hash[ 'name2' ] || feed_hash[ 'subtitle'], - includes: feed_hash[ 'includes' ] || feed_hash[ 'include' ], - excludes: feed_hash[ 'excludes' ] || feed_hash[ 'exclude' ], - author: feed_hash[ 'author' ] || feed_hash[ 'owner' ], - email: feed_hash[ 'email' ], - avatar: feed_hash[ 'avatar' ] || feed_hash[ 'face'], - location: feed_hash[ 'location' ], - github: feed_hash[ 'github' ], - twitter: feed_hash[ 'twitter' ], - rubygems: feed_hash[ 'rubygems' ], - meetup: feed_hash[ 'meetup' ], ### -- remove from schema - virtual attrib ?? - why? why not?? - } - - feed_attribs[:encoding] = feed_hash['encoding']||feed_hash['charset'] if feed_hash['encoding']||feed_hash['charset'] - -##### -## -# auto-fill; convenience helpers - - if feed_hash['meetup'] -## link/url = http://www.meetup.com/vienna-rb -## feed/feed_url = http://www.meetup.com/vienna-rb/events/rss/vienna.rb/ - - feed_attribs[:url] = "http://www.meetup.com/#{feed_hash['meetup']}" if feed_attribs[:url].nil? - feed_attribs[:feed_url] = "http://www.meetup.com/#{feed_hash['meetup']}/events/rss/#{feed_hash['meetup']}/" if feed_attribs[:feed_url].nil? - end - - if feed_hash['googlegroups'] -## link/url = https://groups.google.com/group/beerdb or -## https://groups.google.com/forum/#!forum/beerdb -## feed/feed_url = https://groups.google.com/forum/feed/beerdb/topics/atom.xml?num=15 - - feed_attribs[:url] = "https://groups.google.com/group/#{feed_hash['googlegroups']}" if feed_attribs[:url].nil? - feed_attribs[:feed_url] = "https://groups.google.com/forum/feed//#{feed_hash['googlegroups']}/topics/atom.xml?num=15" if feed_attribs[:feed_url].nil? - end - - if feed_hash['github'] && feed_hash['github'].index('/') ## e.g. jekyll/jekyll -## link/url = https://github.com/jekyll/jekyll -## feed/feed_url = https://github.com/jekyll/jekyll/commits/master.atom - - feed_attribs[:url] = "https://github.com/#{feed_hash['github']}" if feed_attribs[:url].nil? - feed_attribs[:feed_url] = "https://github.com/#{feed_hash['github']}/commits/master.atom" if feed_attribs[:feed_url].nil? + class Site < ActiveRecord::Base + self.table_name = 'sites' + + has_many :subscriptions + has_many :feeds, through: :subscriptions + has_many :items, through: :feeds + + ################################## + # attribute reader aliases + alias_attr_reader :name, :title # alias for title + + alias_attr_reader :owner_name, :author # alias for author + alias_attr_reader :owner, :author # alias(2) for author + alias_attr_reader :author_name, :author # alias(3) for author + + alias_attr_reader :owner_email, :email # alias for email + alias_attr_reader :author_email, :email # alias(2) for email + + def self.deep_create_or_update_from_hash!(name, config, opts = {}) + ## note: allow (optional) config of site key too + site_key = config['key'] || config['slug'] + if site_key.nil? + ## if no key configured; use (file)name; remove -_ chars + ## e.g. jekyll-meta becomes jekyllmeta etc. + site_key = name.downcase.gsub(/[-_]/, '') + end + + site_rec = Site.find_by_key(site_key) + if site_rec.nil? + site_rec = Site.new + site_rec.key = site_key + end + + site_rec.deep_update_from_hash!(config, opts) + site_rec end - if feed_hash['rubygems'] && feed_attribs[:url].nil? && feed_attribs[:feed_url].nil? -## link/url = http://rubygems.org/gems/jekyll -## feed/feed_url = http://rubygems.org/gems/jekyll/versions.atom - - feed_attribs[:url] = "http://rubygems.org/gems/#{feed_hash['rubygems']}" if feed_attribs[:url].nil? - feed_attribs[:feed_url] = "http://rubygems.org/gems/#{feed_hash['rubygems']}/versions.atom" if feed_attribs[:feed_url].nil? + def deep_update_from_hash!(config, _opts = {}) + logger = LogUtils::Logger.root + + site_attribs = { + title: config['title'] || config['name'], # support either title or name + url: config['source'] || config['url'], # support source or url for source url for auto-update (optional) + author: config['author'] || config['owner'], + email: config['email'], + updated: Time.now ## track last_update via pluto (w/ update_subscription_for fn) + } + + ## note: allow (optional) config of site key too + site_key = config['key'] || config['slug'] + site_attribs[:key] = site_key if site_key + + logger.debug "site_attribs: #{site_attribs.inspect}" + + if new_record? + ## use object_id: site.id and object_type: Site + ## change - model/table/schema!!! + Activity.create!(text: "new site >#{key}< - #{title}") + end + + update!(site_attribs) + + # -- log update activity + ## Activity.create!( text: "update subscriptions for site >#{key}<" ) + + #### todo/fix: + ## double check - how to handle delete + ## feeds might get referenced by other sites + ## cannot just delete feeds; only save to delete join table (subscriptions) + ## check if feed "lingers" on with no reference (to site)??? + + # clean out subscriptions and add again + logger.debug "before site.subscriptions.delete_all - count: #{subscriptions.count}" + # NOTE: use destroy_all NOT delete_all (delete_all tries by default only nullify) + subscriptions.destroy_all + logger.debug "after site.subscriptions.delete_all - count: #{subscriptions.count}" + + config.each do |k, v| + ## todo: downcase key - why ??? why not??? + + # skip "top-level" feed keys e.g. title, etc. or planet planet sections (e.g. planet,defaults) + next if %w[key slug + title name name2 title2 subtitle + source url + include includes exclude excludes + feeds + author owner email + planet defaults].include?(k) + + ### todo/check: + ## check value - must be hash + # check if url or feed_url present + # that is, check for required props/key-value pairs + + feed_key = k.to_s.dup + feed_hash = v + + # todo/fix: use title from feed? + # e.g. fill up auto_title, auto_url, etc. + + feed_attribs = { + feed_url: feed_hash['feed'] || feed_hash['feed_url'] || feed_hash['xml_url'], + url: feed_hash['link'] || feed_hash['url'] || feed_hash['html_url'], + title: feed_hash['title'] || feed_hash['name'], + ## note: title2 no longer supported; use summary or subtitle? + ### title2: feed_hash[ 'title2' ] || feed_hash[ 'name2' ] || feed_hash[ 'subtitle'], + includes: feed_hash['includes'] || feed_hash['include'], + excludes: feed_hash['excludes'] || feed_hash['exclude'], + author: feed_hash['author'] || feed_hash['owner'], + email: feed_hash['email'], + avatar: feed_hash['avatar'] || feed_hash['face'], + location: feed_hash['location'], + github: feed_hash['github'], + twitter: feed_hash['twitter'], + rubygems: feed_hash['rubygems'], + meetup: feed_hash['meetup'] ### -- remove from schema - virtual attrib ?? - why? why not?? + } + + if feed_hash['encoding'] || feed_hash['charset'] + feed_attribs[:encoding] = + feed_hash['encoding'] || feed_hash['charset'] + end + + ##### + ## + # auto-fill; convenience helpers + + if feed_hash['meetup'] + ## link/url = http://www.meetup.com/vienna-rb + ## feed/feed_url = http://www.meetup.com/vienna-rb/events/rss/vienna.rb/ + + feed_attribs[:url] = "http://www.meetup.com/#{feed_hash['meetup']}" if feed_attribs[:url].nil? + if feed_attribs[:feed_url].nil? + feed_attribs[:feed_url] = + "http://www.meetup.com/#{feed_hash['meetup']}/events/rss/#{feed_hash['meetup']}/" + end + end + + if feed_hash['googlegroups'] + ## link/url = https://groups.google.com/group/beerdb or + ## https://groups.google.com/forum/#!forum/beerdb + ## feed/feed_url = https://groups.google.com/forum/feed/beerdb/topics/atom.xml?num=15 + + if feed_attribs[:url].nil? + feed_attribs[:url] = + "https://groups.google.com/group/#{feed_hash['googlegroups']}" + end + if feed_attribs[:feed_url].nil? + feed_attribs[:feed_url] = + "https://groups.google.com/forum/feed//#{feed_hash['googlegroups']}/topics/atom.xml?num=15" + end + end + + if feed_hash['github']&.index('/') ## e.g. jekyll/jekyll + ## link/url = https://github.com/jekyll/jekyll + ## feed/feed_url = https://github.com/jekyll/jekyll/commits/master.atom + + feed_attribs[:url] = "https://github.com/#{feed_hash['github']}" if feed_attribs[:url].nil? + if feed_attribs[:feed_url].nil? + feed_attribs[:feed_url] = + "https://github.com/#{feed_hash['github']}/commits/master.atom" + end + end + + if feed_hash['rubygems'] && feed_attribs[:url].nil? && feed_attribs[:feed_url].nil? + ## link/url = http://rubygems.org/gems/jekyll + ## feed/feed_url = http://rubygems.org/gems/jekyll/versions.atom + + feed_attribs[:url] = "http://rubygems.org/gems/#{feed_hash['rubygems']}" if feed_attribs[:url].nil? + if feed_attribs[:feed_url].nil? + feed_attribs[:feed_url] = + "http://rubygems.org/gems/#{feed_hash['rubygems']}/versions.atom" + end + end + + logger.info "Updating feed subscription >#{feed_key}< - >#{feed_attribs[:feed_url]}<..." + + feed_rec = Feed.find_by_key(feed_key) + if feed_rec.nil? + feed_rec = Feed.new + feed_attribs[:key] = feed_key + + ## use object_id: feed.id and object_type: Feed + ## change - model/table/schema!!! + ## todo: add parent_action_id - why? why not? + Activity.create!(text: "new feed >#{feed_key}< - #{feed_attribs[:title]}") + end + + feed_rec.update!(feed_attribs) + + # add subscription record + # note: subscriptions get cleaned out on update first (see above) + subscriptions.create!(feed_id: feed_rec.id) + end end - - - logger.info "Updating feed subscription >#{feed_key}< - >#{feed_attribs[:feed_url]}<..." - - feed_rec = Feed.find_by_key( feed_key ) - if feed_rec.nil? - feed_rec = Feed.new - feed_attribs[:key] = feed_key - - ## use object_id: feed.id and object_type: Feed - ## change - model/table/schema!!! - ## todo: add parent_action_id - why? why not? - Activity.create!( text: "new feed >#{feed_key}< - #{feed_attribs[:title]}" ) - end - - feed_rec.update!( feed_attribs ) - - # add subscription record - # note: subscriptions get cleaned out on update first (see above) - subscriptions.create!( feed_id: feed_rec.id ) end - - end # method deep_update_from_hash! - -end # class Site - - - end # module Model -end # module Pluto + end +end diff --git a/pluto-models/lib/pluto/models/subscription.rb b/pluto-models/lib/pluto/models/subscription.rb index 4b28d2e..fce048a 100644 --- a/pluto-models/lib/pluto/models/subscription.rb +++ b/pluto-models/lib/pluto/models/subscription.rb @@ -1,16 +1,12 @@ -# encoding: utf-8 +# frozen_string_literal: true module Pluto module Model + class Subscription < ActiveRecord::Base + self.table_name = 'subscriptions' - -class Subscription < ActiveRecord::Base - self.table_name = 'subscriptions' - - belongs_to :site - belongs_to :feed + belongs_to :site + belongs_to :feed + end + end end - - - end # module Model -end # module Pluto diff --git a/pluto-models/lib/pluto/models/utils.rb b/pluto-models/lib/pluto/models/utils.rb index 558fcf2..13a442b 100644 --- a/pluto-models/lib/pluto/models/utils.rb +++ b/pluto-models/lib/pluto/models/utils.rb @@ -1,49 +1,43 @@ -# encoding: utf-8 +# frozen_string_literal: true module Pluto module Model + class ItemCursor + def initialize(items) + @items = items + end -class ItemCursor + def each + last_updated = Time.local(1970, 1, 1) + last_feed_id = -1 ## todo: use feed_key instead of id?? why? why not?? - def initialize( items ) - @items = items - end + @items.each do |item| + item_updated = item.updated # cache updated value ref - def each - last_updated = Time.local( 1970, 1, 1 ) - last_feed_id = -1 ## todo: use feed_key instead of id?? why? why not?? + new_date = if last_updated.year == item_updated.year && + last_updated.month == item_updated.month && + last_updated.day == item_updated.day + false + else + true + end - @items.each do |item| + ## note: + # new date also **always** starts new feed + # - e.g. used for grouping within day (follows planet planet convention) - item_updated = item.updated # cache updated value ref + new_feed = if new_date || last_feed_id != item.feed.id + true + else + false + end - if last_updated.year == item_updated.year && - last_updated.month == item_updated.month && - last_updated.day == item_updated.day - new_date = false - else - new_date = true - end + yield(item, new_date, new_feed) -## note: -# new date also **always** starts new feed -# - e.g. used for grouping within day (follows planet planet convention) - - if new_date || last_feed_id != item.feed.id - new_feed = true - else - new_feed = false + last_updated = item.updated + last_feed_id = item.feed.id + end end - - yield( item, new_date, new_feed ) - - last_updated = item.updated - last_feed_id = item.feed.id end - end # method each - -end # class ItemCursor - - - end # module Model -end # module Pluto + end +end diff --git a/pluto-models/lib/pluto/schema.rb b/pluto-models/lib/pluto/schema.rb index 9e87f4a..a596983 100644 --- a/pluto-models/lib/pluto/schema.rb +++ b/pluto-models/lib/pluto/schema.rb @@ -1,153 +1,139 @@ -# encoding: utf-8 +# frozen_string_literal: true module Pluto + class CreateDb + def up + ActiveRecord::Schema.define do + create_table :sites do |t| + t.string :key, null: false # e.g. ruby, js, etc. + t.string :title, null: false # e.g Planet Ruby, Planet JavaScript, etc. + + t.string :author # owner_name, author_name + t.string :email # owner_email, author_email + t.datetime :updated # date for subscription list last updated via pluto + + ############ + # filters (site-wide) + t.string :includes # regex + t.string :excludes # regex + + ###################### + # for auto-update of feed list/site config + + t.string :url # source url for auto-update (optional) + + ## note: make sure to use same fields for update check as feed + + t.datetime :fetched # date for last fetched/checked for feeds via pluto -- make not null ?? + t.integer :http_code # last http status code e.g. 200,404,etc. + t.string :http_etag # last http header etag + ## note: save last-modified header as text (not datetime) - pass through as is + t.string :http_last_modified # last http header last-modified - note: save header as plain text!!! pass along in next request as-is + t.string :http_server # last http server header if present + + # NOTE: do NOT store body content (that is, text) and md5 digest + # use git! and github! commit will be http_etag!! + t.string :md5 # md5 hash of body + + ############# + # more fields + + t.timestamps # created_at, updated_at + end + + create_table :subscriptions do |t| # has_many join table (sites/feeds) + t.references :site, null: false + t.references :feed, null: false + t.timestamps + end + + create_table :feeds do |t| + t.string :key, null: false + t.string :encoding, null: false, default: 'utf8' # charset encoding; default to utf8 + t.string :format # e.g. atom (1.0), rss 2.0, etc. + + t.string :title # user supplied title + t.string :url # user supplied site url + t.string :feed_url # user supplied feed url + + t.string :auto_title # "fallback" - auto(fill) title from feed + t.string :auto_url # "fallback" - auto(fill) url from feed + t.string :auto_feed_url # "fallback" - auto discovery feed url from (site) url + + t.text :summary # e.g. description (rss), subtitle (atom) + ## todo: add auto_summary - why? why not? + + t.string :generator # feed generator (e.g. wordpress, etc.) from feed + + t.datetime :updated # from feed updated(atom) + lastBuildDate(rss) + t.datetime :published # from feed published(atom) + pubDate(rss) - note: published basically an alias for created + + ### extras (move to array for custom fields or similar??) + t.string :author # author_name, owner_name + t.string :email # author_email, owner_email + t.string :avatar # gravator or hackergotchi handle (optional) + t.string :location # e.g. Vienna > Austria, Bamberg > Germany etc. (optional) + + t.string :github # github handle (optional) + t.string :rubygems # rubygems handle (optional) + t.string :twitter # twitter handle (optional) + t.string :meetup # meetup handle (optional) + + ### add class/kind field e.g. + # - personal feed/blog/site, that is, individual author + # - team blog/site + # - org (anization) or com(pany blog/site) + # - newsfeed (composite) + # - other (link blog?, podcast?) - why? why not?? + + ############ + # filters (feed-wide) + t.string :includes # regex + t.string :excludes # regex + # todo: add generic filter list e.g. t.string :filters (comma,pipe or space separated method names?) + + # -- our own (meta) fields + t.datetime :items_last_updated # cache last (latest) updated for items - e.g. latest date from updated item + t.datetime :fetched # last fetched date via pluto + + t.integer :http_code # last http status code e.g. 200,404,etc. + t.string :http_etag # last http header etag + ## note: save last-modified header as text (not datetime) - pass through as is + t.string :http_last_modified # last http header last-modified - note: save header as plain text!!! pass along in next request as-is + t.string :http_server # last http server header if present + + t.string :md5 # md5 hash of body + t.text :body # last http response body (complete feed!) + + t.timestamps # created_at, updated_at + end -class CreateDb + create_table :items do |t| + t.string :guid + t.string :url -def up + ## note: title may contain more than 255 chars!! + ## e.g. Rails Girls blog has massive titles in feed + ## cut-off/limit to 255 - why?? why not?? + ## also strip tags in titles - why? why not?? - see feed.title2/auto_title2 -ActiveRecord::Schema.define do - create_table :sites do |t| - t.string :key, null: false # e.g. ruby, js, etc. - t.string :title, null: false # e.g Planet Ruby, Planet JavaScript, etc. - - t.string :author # owner_name, author_name - t.string :email # owner_email, author_email - t.datetime :updated # date for subscription list last updated via pluto - - ############ - # filters (site-wide) - t.string :includes # regex - t.string :excludes # regex - - - ###################### - # for auto-update of feed list/site config - - t.string :url # source url for auto-update (optional) - - ## note: make sure to use same fields for update check as feed - - t.datetime :fetched # date for last fetched/checked for feeds via pluto -- make not null ?? - t.integer :http_code # last http status code e.g. 200,404,etc. - t.string :http_etag # last http header etag - ## note: save last-modified header as text (not datetime) - pass through as is - t.string :http_last_modified # last http header last-modified - note: save header as plain text!!! pass along in next request as-is - t.string :http_server # last http server header if present - - # note: do NOT store body content (that is, text) and md5 digest - # use git! and github! commit will be http_etag!! - t.string :md5 # md5 hash of body - - - ############# - # more fields - - t.timestamps # created_at, updated_at + t.text :title # TODO: add some :null => false ?? + t.text :summary # e.g. description (rss), summary (atom) + t.text :content + + t.datetime :updated # from feed updated (atom) + pubDate(rss) + t.datetime :published # from feed published (atom) -- note: published is basically an alias for created + + ## todo: add :last_updated_at ?? (NOTE: updated_at already take by auto-timestamps) + t.references :feed, null: false + + t.datetime :fetched # last fetched/check date via pluto + t.timestamps # created_at, updated_at + + ## t.string :author + ## todo: add author/authors, category/categories + end + end end - - - create_table :subscriptions do |t| # has_many join table (sites/feeds) - t.references :site, null: false - t.references :feed, null: false - t.timestamps - end - - create_table :feeds do |t| - t.string :key, null: false - t.string :encoding, null: false, default: 'utf8' # charset encoding; default to utf8 - t.string :format # e.g. atom (1.0), rss 2.0, etc. - - t.string :title # user supplied title - t.string :url # user supplied site url - t.string :feed_url # user supplied feed url - - t.string :auto_title # "fallback" - auto(fill) title from feed - t.string :auto_url # "fallback" - auto(fill) url from feed - t.string :auto_feed_url # "fallback" - auto discovery feed url from (site) url - - t.text :summary # e.g. description (rss), subtitle (atom) - ## todo: add auto_summary - why? why not? - - t.string :generator # feed generator (e.g. wordpress, etc.) from feed - - t.datetime :updated # from feed updated(atom) + lastBuildDate(rss) - t.datetime :published # from feed published(atom) + pubDate(rss) - note: published basically an alias for created - - - ### extras (move to array for custom fields or similar??) - t.string :author # author_name, owner_name - t.string :email # author_email, owner_email - t.string :avatar # gravator or hackergotchi handle (optional) - t.string :location # e.g. Vienna > Austria, Bamberg > Germany etc. (optional) - - t.string :github # github handle (optional) - t.string :rubygems # rubygems handle (optional) - t.string :twitter # twitter handle (optional) - t.string :meetup # meetup handle (optional) - - - ### add class/kind field e.g. - # - personal feed/blog/site, that is, individual author - # - team blog/site - # - org (anization) or com(pany blog/site) - # - newsfeed (composite) - # - other (link blog?, podcast?) - why? why not?? - - ############ - # filters (feed-wide) - t.string :includes # regex - t.string :excludes # regex - # todo: add generic filter list e.g. t.string :filters (comma,pipe or space separated method names?) - - # -- our own (meta) fields - t.datetime :items_last_updated # cache last (latest) updated for items - e.g. latest date from updated item - t.datetime :fetched # last fetched date via pluto - - t.integer :http_code # last http status code e.g. 200,404,etc. - t.string :http_etag # last http header etag - ## note: save last-modified header as text (not datetime) - pass through as is - t.string :http_last_modified # last http header last-modified - note: save header as plain text!!! pass along in next request as-is - t.string :http_server # last http server header if present - - t.string :md5 # md5 hash of body - t.text :body # last http response body (complete feed!) - - - t.timestamps # created_at, updated_at - end - - - create_table :items do |t| - t.string :guid - t.string :url - - ## note: title may contain more than 255 chars!! - ## e.g. Rails Girls blog has massive titles in feed - ## cut-off/limit to 255 - why?? why not?? - ## also strip tags in titles - why? why not?? - see feed.title2/auto_title2 - - t.text :title # todo: add some :null => false ?? - t.text :summary # e.g. description (rss), summary (atom) - t.text :content - - t.datetime :updated # from feed updated (atom) + pubDate(rss) - t.datetime :published # from feed published (atom) -- note: published is basically an alias for created - - ## todo: add :last_updated_at ?? (NOTE: updated_at already take by auto-timestamps) - t.references :feed, null: false - - t.datetime :fetched # last fetched/check date via pluto - t.timestamps # created_at, updated_at - - ## t.string :author - ## todo: add author/authors, category/categories - end - -end # block Schema.define - -end # method up - -end # class CreateDb - -end # module Pluto + end +end diff --git a/pluto-models/lib/pluto/version.rb b/pluto-models/lib/pluto/version.rb index 47a03fb..e085ed9 100644 --- a/pluto-models/lib/pluto/version.rb +++ b/pluto-models/lib/pluto/version.rb @@ -1,11 +1,10 @@ -# encoding: utf-8 +# frozen_string_literal: true module Pluto - MAJOR = 1 MINOR = 6 PATCH = 2 - VERSION = [MAJOR,MINOR,PATCH].join('.') + VERSION = [MAJOR, MINOR, PATCH].join('.') def self.version VERSION @@ -18,13 +17,12 @@ def self.banner ## Note: moved from pluto-merge (add here because pluto-merge gem is optional) ## fix: remove generator in pluto-merge!!! (duplicate) - def self.generator # convenience alias for banner (matches HTML generator meta tag) + # convenience alias for banner (matches HTML generator meta tag) + def self.generator "Pluto #{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]" end - def self.root - "#{File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )}" + File.expand_path(File.dirname(File.dirname(File.dirname(__FILE__)))).to_s end - -end # module Pluto +end diff --git a/pluto-models/test/helper.rb b/pluto-models/test/helper.rb index 50c388d..8f766a8 100644 --- a/pluto-models/test/helper.rb +++ b/pluto-models/test/helper.rb @@ -1,14 +1,14 @@ +# frozen_string_literal: true + ## minitest setup require 'minitest/autorun' ## our own code require 'pluto/models' - Pluto.config.debug = true Pluto.config.logger.level = :debug - ## add some (ActiveRecord model class) shortcuts Log = LogDb::Model::Log Prop = ConfDb::Model::Prop @@ -18,5 +18,4 @@ Item = Pluto::Model::Item Subscription = Pluto::Model::Subscription - Pluto.setup_in_memory_db diff --git a/pluto-models/test/test_delete_removed.rb b/pluto-models/test/test_delete_removed.rb index 4017968..4b6ed41 100644 --- a/pluto-models/test/test_delete_removed.rb +++ b/pluto-models/test/test_delete_removed.rb @@ -1,14 +1,13 @@ +# frozen_string_literal: true + ### # to run use # ruby -I ./lib -I ./test test/test_delete_removed.rb - require 'helper' -class TestDeleteRemoved < MiniTest::Test - +class TestDeleteRemoved < Minitest::Test def test_delete_removed - feed = Feed.create!( key: 'test', title: 'Feed title' @@ -37,11 +36,10 @@ def test_delete_removed feed_data.items << item_data - feed.deep_update_from_struct!( feed_data ) + feed.deep_update_from_struct!(feed_data) assert_equal(3, Item.count) - feed_data.items = [] item_data = FeedParser::Item.new @@ -56,8 +54,8 @@ def test_delete_removed item_data.published = Time.now - 20.minutes feed_data.items << item_data - feed.deep_update_from_struct!( feed_data ) + feed.deep_update_from_struct!(feed_data) assert_equal(2, Item.count) end -end # class TestDeleteRemoved +end diff --git a/pluto-models/test/test_filter.rb b/pluto-models/test/test_filter.rb index f03fe1a..59eaf66 100644 --- a/pluto-models/test/test_filter.rb +++ b/pluto-models/test/test_filter.rb @@ -1,14 +1,13 @@ +# frozen_string_literal: true + ### # to run use # ruby -I ./lib -I ./test test/test_filter.rb - require 'helper' -class TestFilter < MiniTest::Test - +class TestFilter < Minitest::Test def test_includes - feed1 = Feed.create!( key: 'test', title: 'Test', @@ -40,7 +39,6 @@ def test_includes feed_data.items << item_data - item_data = FeedParser::Item.new item_data.title = 'Test #3' item_data.summary = "Test\nTest\nTest" @@ -49,10 +47,9 @@ def test_includes feed_data.items << item_data - feed1.deep_update_from_struct!( feed_data ) ## check w/ includes - feed2.deep_update_from_struct!( feed_data ) ## check w/o includes + feed1.deep_update_from_struct!(feed_data) ## check w/ includes + feed2.deep_update_from_struct!(feed_data) ## check w/o includes - assert true ## if we get here it should workd + assert true ## if we get here it should workd end - -end # class TestFilter +end diff --git a/pluto-models/test/test_helpers.rb b/pluto-models/test/test_helpers.rb index 589618e..4a6010b 100644 --- a/pluto-models/test/test_helpers.rb +++ b/pluto-models/test/test_helpers.rb @@ -1,12 +1,12 @@ +# frozen_string_literal: true + ### # to run use # ruby -I ./lib -I ./test test/test_helpers.rb - require 'helper' -class TestHelper < MiniTest::Test - +class TestHelper < Minitest::Test def setup Log.delete_all Site.delete_all @@ -18,7 +18,7 @@ def setup def test_banner puts Pluto.banner - assert true ## if we get here it should workd + assert true ## if we get here it should workd end def test_models @@ -34,24 +34,22 @@ def test_models def test_auto_migrate Pluto.auto_migrate! - assert true ## if we get here it should workd + assert true ## if we get here it should workd end - - include TextUtils::HypertextHelper ## e.g. lets us use strip_tags( ht ) + include TextUtils::HypertextHelper ## e.g. lets us use strip_tags( ht ) def test_feed_title2_sanitize -## -# todo: -## strip all tags from title2 -## limit to 255 chars -## e.g. title2 such as this exist + ## + # todo: + ## strip all tags from title2 + ## limit to 255 chars + ## e.g. title2 such as this exist - title2_in = %Q{This is a low-traffic announce-only list for people interested in hearing news about Polymer (http://polymer-project.org). The higher-traffic mailing list for all kinds of discussion is https://groups.google.com/group/polymer-dev} - title2_out = %Q{This is a low-traffic announce-only list for people interested in hearing news about Polymer (http://polymer-project.org). The higher-traffic mailing list for all kinds of discussion is https://groups.google.com/group/polymer-dev} + title2_in = %{This is a low-traffic announce-only list for people interested in hearing news about Polymer (http://polymer-project.org). The higher-traffic mailing list for all kinds of discussion is https://groups.google.com/group/polymer-dev} + title2_out = %{This is a low-traffic announce-only list for people interested in hearing news about Polymer (http://polymer-project.org). The higher-traffic mailing list for all kinds of discussion is https://groups.google.com/group/polymer-dev} - assert_equal title2_out, strip_tags( title2_in ) - assert_equal 229, strip_tags( title2_in )[0...255].length + assert_equal title2_out, strip_tags(title2_in) + assert_equal 229, strip_tags(title2_in)[0...255].length end - -end # class TestHelper +end diff --git a/pluto-models/test/test_regex.rb b/pluto-models/test/test_regex.rb index 036bd35..0518e67 100644 --- a/pluto-models/test/test_regex.rb +++ b/pluto-models/test/test_regex.rb @@ -1,28 +1,27 @@ +# frozen_string_literal: true + ### # to run use # ruby -I ./lib -I ./test test/test_regex.rb - require 'helper' -class TestRegex < MiniTest::Test - +class TestRegex < Minitest::Test FIX_DATE_SLUG_RE = Pluto::Model::Feed::FIX_DATE_SLUG_RE def test_fix_dates - m = FIX_DATE_SLUG_RE.match( ' /news/2019-10-17-growing-ruby-together' ) - assert_equal false, m.nil? + m = FIX_DATE_SLUG_RE.match(' /news/2019-10-17-growing-ruby-together') + assert_equal false, m.nil? assert_equal '2019', m[:year] assert_equal '10', m[:month] assert_equal '17', m[:day] - ## check \b (boundray) - m = FIX_DATE_SLUG_RE.match( ' /news/002019-10-1700-growing-ruby-together' ) + ## check \b (boundray) + m = FIX_DATE_SLUG_RE.match(' /news/002019-10-1700-growing-ruby-together') assert_equal true, m.nil? - m = FIX_DATE_SLUG_RE.match( ' /news/2019-10-1700-growing-ruby-together' ) + m = FIX_DATE_SLUG_RE.match(' /news/2019-10-1700-growing-ruby-together') assert_equal true, m.nil? - m = FIX_DATE_SLUG_RE.match( ' /news/xxx2019-10-1700-growing-ruby-together' ) + m = FIX_DATE_SLUG_RE.match(' /news/xxx2019-10-1700-growing-ruby-together') assert_equal true, m.nil? end - -end # class TestRegex +end diff --git a/pluto-models/test/test_site.rb b/pluto-models/test/test_site.rb index 1825802..1173fd6 100644 --- a/pluto-models/test/test_site.rb +++ b/pluto-models/test/test_site.rb @@ -1,12 +1,12 @@ +# frozen_string_literal: true + ### # to run use # ruby -I ./lib -I ./test test/test_site.rb - require 'helper' -class TestSite < MiniTest::Test - +class TestSite < Minitest::Test def setup Log.delete_all Site.delete_all @@ -15,57 +15,53 @@ def setup Item.delete_all end - def test_site_create - site_text = File.read( "#{Pluto::Test.data_dir}/ruby.ini") - site_config = INI.load( site_text ) + site_text = File.read("#{Pluto::Test.data_dir}/ruby.ini") + site_config = INI.load(site_text) ## pp site_config assert_equal 0, Site.count assert_equal 0, Feed.count - Site.deep_create_or_update_from_hash!( 'ruby', site_config ) + Site.deep_create_or_update_from_hash!('ruby', site_config) assert_equal 1, Site.count assert_equal 2, Feed.count - ruby = Site.find_by_key!( 'ruby' ) + ruby = Site.find_by_key!('ruby') assert_equal 'Planet Ruby', ruby.title assert_equal 2, ruby.subscriptions.count assert_equal 2, ruby.feeds.count - rubylang = Feed.find_by_key!( 'rubylang' ) + rubylang = Feed.find_by_key!('rubylang') assert_equal 'Ruby Lang News', rubylang.title assert_equal 'http://www.ruby-lang.org/en/news', rubylang.url assert_equal 'http://www.ruby-lang.org/en/feeds/news.rss', rubylang.feed_url end - def test_site_update - site_text = File.read( "#{Pluto::Test.data_dir}/ruby.ini") - site_config = INI.load( site_text ) + site_text = File.read("#{Pluto::Test.data_dir}/ruby.ini") + site_config = INI.load(site_text) ## pp site_config assert_equal 0, Site.count assert_equal 0, Feed.count ## note: call twice (first time for create, second time for update) - Site.deep_create_or_update_from_hash!( 'ruby', site_config ) - Site.deep_create_or_update_from_hash!( 'ruby', site_config ) + Site.deep_create_or_update_from_hash!('ruby', site_config) + Site.deep_create_or_update_from_hash!('ruby', site_config) assert_equal 1, Site.count assert_equal 2, Feed.count - ruby = Site.find_by_key!( 'ruby' ) + ruby = Site.find_by_key!('ruby') assert_equal 'Planet Ruby', ruby.title assert_equal 2, ruby.subscriptions.count assert_equal 2, ruby.feeds.count - rubylang = Feed.find_by_key!( 'rubylang' ) + rubylang = Feed.find_by_key!('rubylang') assert_equal 'Ruby Lang News', rubylang.title assert_equal 'http://www.ruby-lang.org/en/news', rubylang.url assert_equal 'http://www.ruby-lang.org/en/feeds/news.rss', rubylang.feed_url end - - -end # class TestSite +end