Merge pull request #31 from caciviclab/issue_27

Process Form 497 Page 2 (Late Expenditures)

build/referendum/4/supporting/index.json

@@ -10,7 +10,7 @@
       "id": 2,
       "name": "Causa Justa :: Just Cause (nonprofit 501(c)(3))",
       "payee": "Causa Justa :: Just Cause (nonprofit 501(c)(3))",
-      "amount": 33930.57
+      "amount": 94470.86
     }
   ],
   "total_contributions": 456,

calculators/candidate_expenditures_by_type.rb

@@ -65,7 +65,21 @@ def expenditures_by_candidate_by_type
         ORDER BY "Expn_Code", "Filer_ID", "Report_Num" DESC
       SQL
 
-      results.each do |result|
+      # 497 does not contain "Expn_Code" making this calculator pretty useless
+      # for those contributions.
+      # To make the numbers line up closer, we'll bucket those all under "Not
+      # Stated".
+      late_expenditures = ActiveRecord::Base.connection.execute(<<-SQL)
+        SELECT DISTINCT ON ("Filer_ID")
+          "Filer_ID", '' AS "Expn_Code", SUM("Amount") AS "Total"
+        FROM "efile_COAK_2016_497"
+        WHERE "Filer_ID" IN ('#{@candidates_by_filer_id.keys.join "','"}')
+        AND "Form_Type" = 'F497P2'
+        GROUP BY "Filer_ID", "Report_Num"
+        ORDER BY "Filer_ID", "Report_Num" DESC
+      SQL
+
+      (results.to_a + late_expenditures.to_a).each do |result|
         hash[result['Filer_ID']] ||= {}
         hash[result['Filer_ID']][result['Expn_Code']] = result['Total']
       end

calculators/referendum_supporters_calculator.rb

@@ -16,6 +16,16 @@ def fetch
       ORDER BY "Filer_ID", "Filer_NamL", "Bal_Name", "Sup_Opp_Cd", "Report_Num" DESC
     SQL
 
+    late_expenditures = ActiveRecord::Base.connection.execute(<<-SQL)
+      SELECT DISTINCT ON ("Filer_ID", "Filer_NamL", "Bal_Name")
+        "Filer_ID", "Filer_NamL", "Bal_Name", SUM("Amount") AS "Total_Amount"
+      FROM "efile_COAK_2016_497"
+      WHERE "Bal_Name" IS NOT NULL
+      AND "Form_Type" = 'F497P2'
+      GROUP BY "Filer_ID", "Filer_NamL", "Bal_Name", "Report_Num"
+      ORDER BY "Filer_ID", "Filer_NamL", "Bal_Name", "Report_Num" DESC
+    SQL
+
     supporting_by_measure_name = {}
     opposing_by_measure_name = {}
 
@@ -29,6 +39,35 @@ def fetch
       end
     end
 
+    late_expenditures.each do |row|
+      sup_opp_cd = guess_whether_committee_supports_measure(row['Filer_ID'], row['Bal_Name'])
+      if sup_opp_cd == 'S'
+        supporting_by_measure_name[row['Bal_Name']] ||= []
+        existing_idx = supporting_by_measure_name[row['Bal_Name']].find_index do |existing_row|
+          existing_row['Filer_ID'].to_s == row['Filer_ID'].to_s
+        end
+
+        if existing_idx
+          supporting_by_measure_name[row['Bal_Name']][existing_idx]['Total_Amount'] +=
+            row['Total_Amount']
+        else
+          supporting_by_measure_name[row['Bal_Name']] << row
+        end
+      elsif sup_opp_cd == 'O'
+        opposing_by_measure_name[row['Bal_Name']] ||= []
+        existing_idx = opposing_by_measure_name[row['Bal_Name']].find_index do |existing_row|
+          existing_row['Filer_ID'].to_s == row['Filer_ID'].to_s
+        end
+
+        if existing_idx
+          opposing_by_measure_name[row['Bal_Name']][existing_idx]['Total_Amount'] +=
+            row['Total_Amount']
+        else
+          opposing_by_measure_name[row['Bal_Name']] << row
+        end
+      end
+    end
+
     [
       # { bal_name => rows }     , calculation name
       [supporting_by_measure_name, :supporting_organizations],
@@ -77,4 +116,29 @@ def committee_from_expenditure(expenditure)
 
     committee
   end
+
+  # Form 497 Page 2 (Late Expenditures) includes the ballot measure name and
+  # committee ID, but does not indicate whether that expenditure was in support
+  # or opposition of the ballot measure.
+  #
+  # This is not perfect, but it should get us pretty close.
+  def guess_whether_committee_supports_measure(committee_id, bal_name)
+    @_guess_cache ||=
+      begin
+        guesses = ActiveRecord::Base.connection.execute(<<-SQL)
+          SELECT "Filer_ID", "Bal_Name", "Sup_Opp_Cd"
+          FROM "efile_COAK_2016_E-Expenditure"
+          WHERE "Bal_Name" IS NOT NULL
+          GROUP BY "Filer_ID", "Bal_Name", "Sup_Opp_Cd"
+        SQL
+
+        guesses.index_by do |row|
+          row.values_at('Filer_ID', 'Bal_Name').map(&:to_s)
+        end
+      end
+
+    if row = @_guess_cache[[committee_id.to_s, bal_name]]
+      row['Sup_Opp_Cd']
+    end
+  end
 end

calculators/total_expenditures_calculator.rb

@@ -5,7 +5,7 @@ def initialize(candidates: [], ballot_measures: [], committees: [])
   end
 
   def fetch
-    @results = ActiveRecord::Base.connection.execute <<-SQL
+    results = ActiveRecord::Base.connection.execute <<-SQL
       SELECT DISTINCT ON ("Filer_ID", "Amount_A")
         "Filer_ID", "Amount_A"
       FROM "efile_COAK_2016_Summary"
@@ -16,7 +16,17 @@ def fetch
       ORDER BY "Filer_ID", "Amount_A", "Report_Num" DESC
     SQL
 
-    @results.each do |result|
+    late_expenditures = ActiveRecord::Base.connection.execute <<-SQL
+      SELECT DISTINCT ON ("Filer_ID")
+        "Filer_ID", SUM("Amount") AS "Amount_A"
+      FROM "efile_COAK_2016_497"
+      WHERE "Filer_ID" IN ('#{@candidates_by_filer_id.keys.join "', '"}')
+      AND "Form_Type" = 'F497P2'
+      GROUP BY "Filer_ID", "Report_Num"
+      ORDER BY "Filer_ID", "Report_Num" DESC
+    SQL
+
+    (results.to_a + late_expenditures.to_a).each do |result|
       candidate = @candidates_by_filer_id[result['Filer_ID'].to_i]
       candidate.save_calculation(:total_expenditures, result['Amount_A'])
     end