# cars like what

# To run this you need 

# -- this file and weights_approach_matrices.r in the same directory
source("weights_approach_functions.r")

# -- a directory at path_to_company_data (defined in the next line) . . . 
path_to_company_data = "./../../output/weights_approach/quote_matrices/" # change this if running on another machine
# . . . in which you have 
# trading_dates.csv -- trading dates in 2004-2008
# company_ids.csv -- the set of company ids (internal to project) in our data 
# holding_returns_2003_2009_trimmed.csv -- daily returns from CRSP, holding portfolio
# returns_longer_trimmed.csv -- daily returngs from CRSP, excluding distributions
# returns_indices_longer_2.csv -- an index value for each company built by compoundiong the holding returns
# vwretx_2003_2009.csv -- value weighted returns, no distributions -- is that the right index?  

# -- a directory at path_to_connection_data (defined in the next line) . . . 
path_to_connection_data = "./../../output/weights_approach/connection_matrices/" # change this if running on another machine
# . . . in which you have 
# committee_lobbying.csv
# contributions.csv
# in_district_or_state.csv  
# optional: in_state.csv
# optional: committee_jurisdiction.csv

# -- a directory of sparse transactions matrices: we output for each member a csv where each row is a transaction, with company_id, date, and amount 
path_to_transactions_matrices = "./../../output/weights_approach/assets_and_transactions_matrices/"

path_to_output = "./../../output/weights_approach/returns/cars/"


get.big.stuff = TRUE  # a way to avoid getting the big connection matrices once they are already loaded in 
holding.returns = TRUE  # we use the returns with distributions 
if(holding.returns == TRUE){holding.bit = "_holding"}else{holding.bit = "_straight"}

if(get.big.stuff == TRUE){
	cat("Getting big matrices.")

	vwrets = read.csv(paste(path_to_company_data, "vwretd_and_vwretx_2003_2009.csv", sep = ""), header = T)  # this needs to be longer
	vwrets[,1] = as.Date(as.character(vwrets[,1]), format = "%Y%m%d")

	if(holding.returns == TRUE){
	  Q = read.csv(paste(path_to_company_data, "holding_returns_2003_2009_trimmed.csv", sep = ""), row.names = 1, header = T)
	  market.index = vwrets[,c(1:2)] # gets vwretd
	}else{Q = read.csv(paste_csv(path_to_company_data, "returns_2003_2009_trimmed.csv", sep = ""), row.names = 1, header = T)
	  market.index = vwrets[,c(1:3)] # gets vwretx 
	}
	
	cat(".")
	all.company.ids = as.integer(sub("retx", "", colnames(Q)))
	cat(".")
	cat(".\n")

	# this is exactly from member_by_member_connection_portfolio_returns -- could try to DRY it up
	cat("Setting up connection matrices.")
	comm.lobbying.mat = read.csv(paste(path_to_connection_data, "committee_lobbying.csv", sep = ""), header = T, row.names = 1)
	comm.lobbying.any.mat = comm.lobbying.mat > 0 # that's good
	# now we want a vector of members' percentile values
	committee.lobbying.mat2 = comm.lobbying.mat
	committee.lobbying.mat2[comm.lobbying.mat == 0] = NA
	member.lobbying.2 = apply(committee.lobbying.mat2, 1, quantile, .8, na.rm = T)
	member.lobbying.2[is.na(member.lobbying.2)] = 0
	member.lobbying.5 = apply(committee.lobbying.mat2, 1, quantile, .5, na.rm = T)
	member.lobbying.5[is.na(member.lobbying.5)] = 0
	comm.lobbying.20.mat = comm.lobbying.mat > member.lobbying.2 
	comm.lobbying.50.mat = comm.lobbying.mat > member.lobbying.5

	cat(".")
	contrib.mat = read.csv(paste(path_to_connection_data, "contributions.csv", sep = ""), header = T, row.names = 1)
	contrib.mat2 = contrib.mat
	contrib.mat2[contrib.mat == 0] = NA # this takes forever
	member.contrib.2 = apply(contrib.mat2, 1, quantile, .8, na.rm = T)
	member.contrib.2[is.na(member.contrib.2)] = 0
	member.contrib.5 = apply(contrib.mat2, 1, quantile, .5, na.rm = T)
	member.contrib.5[is.na(member.contrib.5)] = 0
	contrib.20.mat = contrib.mat > member.contrib.2 
	contrib.50.mat = contrib.mat > member.contrib.5 
	contrib.any.mat = contrib.mat > 0

	cat(".")
	stdist.mat = read.csv(paste(path_to_connection_data, "in_district_or_state.csv", sep = ""), header = T, row.names = 1) #ones and zeros
	
	# combination matrices 
	cat(".")
	stdist.and.contrib.mat = stdist.mat == 1 & contrib.any.mat
	stdist.and.lobbying.mat = stdist.mat == 1 & comm.lobbying.any.mat
	contrib.and.lobbying.mat = contrib.any.mat == 1 & comm.lobbying.any.mat
	cat("\n")
}

for(mapping in c("ziob", "lognormal")){


for (i in 1:650){ # 650){ # 650){
  if(!paste("transactions_", i, "_sp.csv", sep = "") %in% list.files(path_to_transactions_matrices)){next}
  cat(mapping, "member", i, "\n")
  # now for each we need to produce the list of company ids.
  member.row.name = paste("m", i, sep = "")
  cat("L ")
  out = car.set.mat(i, Q = Q, market.index = market.index, masking.vector = as.logical(comm.lobbying.any.mat[member.row.name,]), value.mapping = mapping)
  write.table(out, paste(path_to_output, "committee_lobbying_any_", mapping, "_", holding.bit, "_returns.csv", sep = ""), append = (i != 1), row.names = F, col.names = (i == 1), sep = ",")
  cat("L2 ")
  out = car.set.mat(i, Q = Q, market.index = market.index, masking.vector = as.logical(comm.lobbying.20.mat[member.row.name,]), value.mapping = mapping)
  write.table(out, paste(path_to_output, "committee_lobbying_20thpctile_", mapping, "_", holding.bit, "_returns.csv", sep = ""), append = (i != 1), row.names = F, col.names = (i == 1), sep = ",")
  cat("L5 ")
  out = car.set.mat(i, Q = Q, market.index = market.index, masking.vector = as.logical(comm.lobbying.50.mat[member.row.name,]), value.mapping = mapping)
  write.table(out, paste(path_to_output, "committee_lobbying_50thpctile_", mapping, "_", holding.bit, "_returns.csv", sep = ""), append = (i != 1), row.names = F, col.names = (i == 1), sep = ",")
  cat("C ")
  out = car.set.mat(i, Q = Q, market.index = market.index, masking.vector = as.logical(contrib.any.mat[member.row.name,]), value.mapping = mapping)
  write.table(out, paste(path_to_output, "contrib_any_", mapping, "_", holding.bit, "_returns.csv", sep = ""), append = (i != 1), row.names = F, col.names = (i == 1), sep = ",")
  cat("C2 ")
  out = car.set.mat(i, Q = Q, market.index = market.index, masking.vector = as.logical(contrib.20.mat[member.row.name,]), value.mapping = mapping)
  write.table(out, paste(path_to_output, "contrib_20thpctile_", mapping, "_", holding.bit, "_returns.csv", sep = ""), append = (i != 1), row.names = F, col.names = (i == 1), sep = ",")
  cat("C5 ")
  out = car.set.mat(i, Q = Q, market.index = market.index, masking.vector = as.logical(contrib.50.mat[member.row.name,]), value.mapping = mapping)
  write.table(out, paste(path_to_output, "contrib_50thpctile_", mapping, "_", holding.bit, "_returns.csv", sep = ""), append = (i != 1), row.names = F, col.names = (i == 1), sep = ",")
  cat("D ")
  out = car.set.mat(i, Q = Q, market.index = market.index, masking.vector = as.logical(stdist.mat[member.row.name,]), value.mapping = mapping)
  write.table(out, paste(path_to_output, "state_or_district_", mapping, "_", holding.bit, "_returns.csv", sep = ""), append = (i != 1), row.names = F, col.names = (i == 1), sep = ",")
  cat("DC ")
  out = car.set.mat(i, Q = Q, market.index = market.index, masking.vector = as.logical(stdist.and.contrib.mat[member.row.name,]), value.mapping = mapping)
  write.table(out, paste(path_to_output, "state_or_district_and_contrib_", mapping, "_", holding.bit, "_returns.csv", sep = ""), append = (i != 1), row.names = F, col.names = (i == 1), sep = ",")
  cat("DL ")
  out = car.set.mat(i, Q = Q, market.index = market.index, masking.vector = as.logical(stdist.and.lobbying.mat[member.row.name,]), value.mapping = mapping)
  write.table(out, paste(path_to_output, "state_or_district_and_lobbying_", mapping, "_", holding.bit, "_returns.csv", sep = ""), append = (i != 1), row.names = F, col.names = (i == 1), sep = ",")
  cat("CL ")
  out = car.set.mat(i, Q = Q, market.index = market.index, masking.vector = as.logical(contrib.and.lobbying.mat[member.row.name,]), value.mapping = mapping)
  write.table(out, paste(path_to_output, "contrib_and_lobbying_", mapping, "_", holding.bit, "_returns.csv", sep = ""), append = (i != 1), row.names = F, col.names = (i == 1), sep = ",")
}
}