function [y,varargout]=backtestcomplex(x,varargin)

% Created by mjf(Tel: 13718883838) on 2012-07-15

% latest justified by mjf on 2012-09-20

% 输入x是一个excel文件的地址字符串，如‘E:\Top50.xlsx’, excel文件的第一行为表头，包含4列：股票交易代码(SZ000001，SH600001形式)，中文简称，选股日期

%(yyyy-mm-dd形式)，权重因子

% 输出y是一个矩阵,有portfolioNum+3列，第一列对应交易日期，2:end-2列对应该交易日期组合的复权收盘价；如果第一列的值为0，那么该行对应的是组合的权重因子；

%   第end-1列为该交易日相对于上个交易日组合的收益率；第end列为策略的累计收益率

% vargin是一个数字或者日期字符串，设定回测的结束日期，默认回测到昨天。

% varargout{varout_infoRatio}是一个structure,有两个field：daily和monthly，分别是一个两列的矩阵，第一列对应日期，第二列表示从该日期起到回测结束策略的信息比率

% varargout{varout_maxDrawDown}是一个size(y,1)-1行2列的矩阵，分别对应组合和市场指数从y(1:end-1,1)交易日开始，到y(end,1)交易日结束时的最大回撤

% varargout{varout_index}是一个structure,有HS300和SZZZ两个field，对应相应的市场指数

% varargout{varout_matketCap}是一个两列的矩阵，第一列为日期，第二列为该日期组合的流通市值；

% varargout{varout_FF3},对应Fama-French三因子回归超额收益alpha

% varargout{varout_xlsResult}输出xlsResult在y中每个组合前面加上了“中文简称”，“交易代码”，“windID”三行，方便写为excel文件

% raw=x;

[~,~,raw]=xlsread(x);

tradeCost=0;

refIndexName='HS300'; %计算信息比率的benchmark

risklessReturn=1+4e-2/365;

weightingMethod=1;% 0:等权重；1：流通市值加权

varout_infoRatio=3;

varout_maxDrawDown=2;

varout_index=1;

varout_matketCap=4;

varout_FF3=5;

varout_xlsResult=6;

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 设定回测结束日期datenum给huicheEndDateNum

if nargin==1 || isempty(varargin)

huicheEndDateNum=today-1; % 设定回测的结束日期，结束日期设为当天之前的1天，就减去1

elseif isnumeric(varargin{1})

huicheEndDateNum=today-varargin{1};

elseif ischar(varargin{1})&&length(varargin{1})==6

huicheEndDateNum=monthEndDateNum(datenum([varargin{1},'01'],'yyyymmdd'));

elseif ischar(varargin{1})&&length(varargin{1})==7

huicheEndDateNum=monthEndDateNum(datenum([varargin{1},'-01'],'yyyy-mm-dd'));

elseif ischar(varargin{1})&&length(varargin{1})==8

huicheEndDateNum=datenum(varargin{1},'yyyymmdd');

elseif ischar(varargin{1})&&length(varargin{1})==10

huicheEndDateNum=datenum(varargin{1},'yyyy-mm-dd');

else

y='Not correct backtest end DATE format.';

fprintf('%s\n',y)

return

end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

format longG

if isnan(raw{1,end})

raw(:,end)=[];

if size(raw,2)~=4

fprintf('Data format may be not correct. Verify it.\n');

return;

end

end

raw=[raw,[{'wind证券ID'};cell(size(raw,1)-1,1)]];

% raw是一个5列的cell array，分别是：股票交易代码，中文简称，选股日期，权重因子，wind证券ID

tradeIDCol=1;

tradeCodeBeginBit=3;

chineseNameCol=2;

dateCol=3;

weightingFactorCol=4;

windIDCol=5;

conn=database('wind','userName','passWord');

%将raw第3列的日期格式转变为与wind数据库相同的'yyyymmdd'形式，且以str类型存储

tmp=datenum(raw(2:end,dateCol),'yyyy-mm-dd');

raw(2:end,dateCol)=num2cell(datestr(tmp,'yyyymmdd'),2);

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 找出每只股票的windID，存入raw的第5列，如果一个组合为空仓，即没有购买股票，这个组合的wingID留空

raw(1,:)=[]; % 去掉raw的表头，即第一行

blankPortfolioRowNum=find(strcmp('AA000000',raw(:,tradeIDCol))); % 约定空仓组合所在行的交易代码为‘AA000000’

blankPortfolio=raw(blankPortfolioRowNum,:);

raw(blankPortfolioRowNum,:)=[];

tradeID=cell2mat(raw(:,tradeIDCol));

tradeID=tradeID(:,tradeCodeBeginBit:end);

tradeID=num2cell(tradeID,2);

windIDAll=strconnec(tradeID);

tmp=fetch(conn,['select f2_1090 from tb_object_1090 where f4_1090=''A'' and f16_1090 in (',windIDAll,') order by f16_1090']);

[~,~,seq]=unique(tradeID);

raw(:,windIDCol)=tmp(seq,:);

raw=[blankPortfolio;raw];

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 找出各个组合在raw中对应的行范围，存储到portfolioRawRowScope，第一列为组合的开始行，第二列为组合的结束行

raw=sortrows(raw,dateCol);

[~,firstRowNum]=unique(raw(:,dateCol),'first');

[~,lastRowNum]=unique(raw(:,dateCol),'last');

portfolioRawRowScope=[firstRowNum,lastRowNum];

totalPortfolioNum=size(portfolioRawRowScope,1);

maxPortfolioCapacity=max(portfolioRawRowScope(:,2)-portfolioRawRowScope(:,1))+1;

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% y现在有maxPortfolioNum+3行，第一行对应交易日期，2:end-2行对应该交易日期组合的收盘价；

% 如果y第一行的值为0，那么该列对应的是组合的权重因子；end-1行为隔日收益率，end行为累计收益率

approxiamteBackTestMonthNum=ceil((huicheEndDateNum-datenum(raw{1,dateCol},'yyyymmdd'))/28);

y=zeros(maxPortfolioCapacity+3,approxiamteBackTestMonthNum*32);

marketCap=zeros(approxiamteBackTestMonthNum*32,2);

xlsResult=cell(maxPortfolioCapacity+3,approxiamteBackTestMonthNum*35);

yCol=0;

marketCapCol=0;

xlsResultCol=0;

for portfolioSN=1:totalPortfolioNum

%     确定该组合的结束日期

if datenum(raw{portfolioRawRowScope(portfolioSN,1),dateCol},'yyyymmdd')>=huicheEndDateNum

break

elseif portfolioSN

endDateStr=raw{portfolioRawRowScope(portfolioSN+1,1),dateCol};

elseif portfolioSNhuicheEndDateNum

endDateStr=datestr(huicheEndDateNum,'yyyymmdd'); % 最新的组合持有到huicheEndDateNum的时间

elseif portfolioSN==totalPortfolioNum

endDateStr=datestr(huicheEndDateNum,'yyyymmdd');

end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%     一个组合是空仓

if isempty(raw{portfolioRawRowScope(portfolioSN,1),windIDCol})

tradeDay=fetch(conn,['select f1_1010 from tb_object_1010 where f1_1010 between ''',...

raw{portfolioRawRowScope(portfolioSN,1),dateCol},''' and ''',endDateStr,''' order by f1_1010']);

yCol=yCol+1;

y(1,yCol+1:yCol+length(tradeDay))=str2double(tradeDay)';

y(end-1,yCol:yCol+length(tradeDay))=1;

yCol=yCol+length(tradeDay);

{portfolioSN,size(portfolioRawRowScope,1),raw{portfolioRawRowScope(portfolioSN,1),dateCol},'short'}

continue;

end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

currentPortfolioCapacity=portfolioRawRowScope(portfolioSN,2)-portfolioRawRowScope(portfolioSN,1)+1; %当前组合中的股票数目

currentPortfolioWindID=raw(portfolioRawRowScope(portfolioSN,1):portfolioRawRowScope(portfolioSN,2),windIDCol);

currentPortfolioWindID=strconnec(currentPortfolioWindID);

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%     权重因子：weightingMethod==0：等权重；weightingMethod==1：流通市值加权

if weightingMethod==0

raw(portfolioRawRowScope(portfolioSN,1):portfolioRawRowScope(portfolioSN,2),weightingFactorCol)={1/currentPortfolioCapacity};

else

%         此处一个隐藏的bug是当一只股票在对应交易日的流通市值没有数据时，会提示错误而中断程序

mktCap=fetch(conn,['select f1_5004,f10_5004 from tb_object_5004 where f2_5004=''',...

raw{portfolioRawRowScope(portfolioSN,1),dateCol},''' and f1_5004 in (',currentPortfolioWindID,')']);

if size(mktCap,1)~=currentPortfolioCapacity

no_mktCap=setdiff(raw(portfolioRawRowScope(portfolioSN,1):portfolioRawRowScope(portfolioSN,2),windIDCol),mktCap(:,1));

tmp=containers.Map(raw(portfolioRawRowScope(portfolioSN,1):portfolioRawRowScope(portfolioSN,2),windIDCol),...

raw(portfolioRawRowScope(portfolioSN,1):portfolioRawRowScope(portfolioSN,2),tradeIDCol));

no_mktCapID=values(tmp,no_mktCap);

fprintf('Can not fetch the Market Capacity for stocks %s on %s\n',strconnec(no_mktCapID),raw{portfolioRawRowScope(portfolioSN,1),dateCol});

no_mktCap=fetch(conn,['select f1_5004,f10_5004 from tb_object_5004 where f2_5004<=''',...

raw{portfolioRawRowScope(portfolioSN,1),dateCol},''' and f1_5004 in (',strconnec(no_mktCap),') order by f2_5004']);

[~,tmp]=unique(no_mktCap(:,1),'last');

no_mktCap=no_mktCap(tmp,:);

mktCap=[mktCap;no_mktCap];

end

cap=cell2mat(mktCap(:,2));

mktCap=containers.Map(mktCap(:,1),cap/sum(cap));

raw(portfolioRawRowScope(portfolioSN,1):portfolioRawRowScope(portfolioSN,2),weightingFactorCol)=...

values(mktCap,raw(portfolioRawRowScope(portfolioSN,1):portfolioRawRowScope(portfolioSN,2),windIDCol));

end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%     SQLserver的 BETWEEN……AND关键词给出的结果包含上下范围的边界

closingPrice=fetch(conn,['select f1_1425,f2_1425,f7_1425 from tb_object_1425 where f1_1425 IN (',currentPortfolioWindID,...

') AND (f2_1425 BETWEEN ''',raw{portfolioRawRowScope(portfolioSN,1),dateCol},''' AND ''',endDateStr,''')']);

% closingPrice是cell array，有三列数据：wind证券ID(char类型)，交易时间(char类型)，复权收盘价(double类型)

% 如果wind数据库中没有一只股票某个交易日的收盘价数据，得到的tmp中也缺乏该股票这个交易日的数据。

% 修正办法：让该股票这个交易日的收盘价等于最近有数据的交易日的收盘价。

% 还有一个bug需要修正：有可能一只股票的数据在组合建立的第一天就没有；此bug已修正

uniqueWindID=sortrows(raw(portfolioRawRowScope(portfolioSN,1):portfolioRawRowScope(portfolioSN,2),windIDCol),1);

tradeSeriesMatrix=col2matrix(closingPrice);

noDataWindID=uniqueWindID(find(1-ismember(uniqueWindID,tradeSeriesMatrix(2:end,1))));

if ~isempty(noDataWindID)

fprintf('Caution: can not fetch closing price data for stock(s) %s from %s to %s. These datas are set to 1.',...

cell2mat(strcat(strcat(noDataWindID,',')')),raw{portfolioRawRowScope(portfolioSN,1),dateCol},endDateStr);

missed=[noDataWindID,num2cell(ones(length(noDataWindID),size(tradeSeriesMatrix,2)-1))]; %所有的交易日内都没有数据，收盘价统一设为1,该股票没有盈亏

tradeSeriesMatrix=[tradeSeriesMatrix(1,:);sortrows([tradeSeriesMatrix(2:end,:);missed],1)];

end

for tradeSeriesMatrixRow=2:size(tradeSeriesMatrix,1)

for tradeSeriesMatrixCol=3:size(tradeSeriesMatrix,2)

if isnan(tradeSeriesMatrix{tradeSeriesMatrixRow,tradeSeriesMatrixCol})

tradeSeriesMatrix{tradeSeriesMatrixRow,tradeSeriesMatrixCol}=tradeSeriesMatrix{tradeSeriesMatrixRow,tradeSeriesMatrixCol-1};

end

end

end

%     股票在组合建立当日就没有数据，收盘价设为此后最近有数据交易日的收盘价

for tradeSeriesMatrixRow=2:size(tradeSeriesMatrix,1)

lastNanCol=find(isnan(cell2mat(tradeSeriesMatrix(tradeSeriesMatrixRow,2:end))),1,'last')+1;

tradeSeriesMatrix(tradeSeriesMatrixRow,2:lastNanCol)=tradeSeriesMatrix(tradeSeriesMatrixRow,lastNanCol+1);

end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

totalTradeDate=size(tradeSeriesMatrix,2)-1;

yCol=yCol+1;

%对raw排序，因为tradeSeriesMatrix对windid做了排序，不然权重因子对不上，add by van 20121227(van.sun.cn@hotmail.com)

tmpArray=sortrows(raw(portfolioRawRowScope(portfolioSN,1):portfolioRawRowScope(portfolioSN,2),:),windIDCol);

y(2:currentPortfolioCapacity+1,yCol)=cell2mat(tmpArray(:,weightingFactorCol));

y(1,yCol+1:yCol+totalTradeDate)=str2double(tradeSeriesMatrix(1,2:end));

y(2:currentPortfolioCapacity+1,yCol+1:yCol+totalTradeDate)=cell2mat(tradeSeriesMatrix(2:end,2:end));

y(end,yCol:yCol+totalTradeDate)=[1,1,y(2:currentPortfolioCapacity+1,yCol)'*...

(y(2:currentPortfolioCapacity+1,yCol+2:yCol+totalTradeDate)./repmat(y(2:currentPortfolioCapacity+1,yCol+1),1,totalTradeDate-1))];

y(end-1,yCol:yCol+totalTradeDate)=[1,1,y(end,yCol+2:yCol+totalTradeDate)./y(end,yCol+1:yCol+totalTradeDate-1)];

y(end-1,yCol+1)=y(end-1,yCol+1)*(1-tradeCost); % 增加买入交易成本

y(end-1,yCol+totalTradeDate)=y(end-1,yCol+totalTradeDate)*(1-tradeCost);% 增加卖出交易成本

yCol=yCol+totalTradeDate;

%%%%%%%%%%%%%%%%%%%%% 流通市值

if nargout>=varout_matketCap+1

currentMarketCap=marketCapFun(raw(portfolioRawRowScope(portfolioSN,1):portfolioRawRowScope(portfolioSN,2),windIDCol),...

raw{portfolioRawRowScope(portfolioSN,1),dateCol},endDateStr,conn);

if size(currentMarketCap,1)

for i=1:size(currentMarketCap,1)

if ~ismember(num2str(currentMarketCap(i,1)),tradeSeriesMatrix(1,2:end))

fprintf('Can not fetch market capitalization data on %d for stocks in portfoio %d when caculating market Cap.\n',...

currentMarketCap(i,1),portfolioSN);

end

end

end

%前一个组合的结束日期和后一个组合的开始日期在同一个交易日，在该日，策略的流通市值取后一个组合的流通市值，因此前一个组合的流通市值只取1:end-1

i=1-sign(str2double(tradeSeriesMatrix(1,end))-currentMarketCap(end,1));

marketCap(marketCapCol+1:marketCapCol+size(currentMarketCap,1)-i,:)=currentMarketCap(1:end-i,:);

marketCapCol=marketCapCol+size(currentMarketCap,1)-i;

end

%%%%%%%%%%%%%%%%%%%%% xlsresult

if nargout>=varout_xlsResult+1

xlsResult(1,xlsResultCol+1:xlsResultCol+4)={'中文简称','交易代码','windID','权重因子'};

xlsResult(2:currentPortfolioCapacity+1,xlsResultCol+1:xlsResultCol+4)=...

sortrows(raw(portfolioRawRowScope(portfolioSN,1):portfolioRawRowScope(portfolioSN,2),[chineseNameCol,tradeIDCol,windIDCol,weightingFactorCol]),3);

xlsResult(:,xlsResultCol+5:xlsResultCol+4+totalTradeDate)=num2cell(y(:,yCol+1:yCol+totalTradeDate));

xlsResult(end-1,xlsResultCol+1:xlsResultCol+4+totalTradeDate)=[1,1,1,num2cell(y(end-1,yCol:yCol+totalTradeDate))];

xlsResultCol=xlsResultCol+4+totalTradeDate;

end

[portfolioSN,size(portfolioRawRowScope,1)]

end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

y(:,yCol+1:end)=[];

if isempty(y)

fprintf('The backtest end-date is prior to the construction-date of the first portfolio.\n');

return;

end

y(end,:)=cumprod(y(end-1,:));

y=y';

spaceRow=find(y(:,1)==0);

y([1;spaceRow(2:end);spaceRow(2:end)+1],:)=[]; %将权重因子所在行全部去掉

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 信息比率information ratio

if nargout>=varout_infoRatio+1

marketIndex=idx(raw{1,dateCol},datestr(huicheEndDateNum,'yyyymmdd'),conn);

if strcmp(refIndexName,'HS300')

refIndex=marketIndex.HS300;

else

refIndex=marketIndex.SZZZ;

end

%     eval(['refIndex=marketIndex.',refIndexName,';']);

if any(refIndex(:,1)~=y(:,1))

fprintf('Caution: Index date does not totally match with y date when caculating Information Ratio.\n');

end

refIndex=refIndex(:,2);

indexDailyReturn=[1;refIndex(2:end)./refIndex(1:end-1)];

yMonth=floor(y(:,1)/100);

[~,firstRowNum]=unique(yMonth,'first');

[~,lastRowNum]=unique(yMonth,'last');

yMonth_YRowScope=[firstRowNum,lastRowNum];

portfolioMonthlyReturn=zeros(size(yMonth_YRowScope,1),1);

indexMonthlyReturn=zeros(size(yMonth_YRowScope,1),1);

for yMonthSN=2:size(yMonth_YRowScope,1)-1

stgMonthSN_YRow=yMonth_YRowScope(yMonthSN,1):yMonth_YRowScope(yMonthSN,2);

portfolioMonthlyReturn(yMonthSN)=prod(y(stgMonthSN_YRow,end-1));

indexMonthlyReturn(yMonthSN)=prod(indexDailyReturn(stgMonthSN_YRow));

end

portfolioMonthlyReturn([1,yMonthSN+1:end])=[];

indexMonthlyReturn([1,yMonthSN+1:end])=[];

monthlyTrackingError=portfolioMonthlyReturn-indexMonthlyReturn;

monthlyIR=zeros(size(monthlyTrackingError,1),2);

for yMonthSN=1:size(monthlyIR,1)-12

monthlyIR(yMonthSN,:)=[yMonth(yMonth_YRowScope(yMonthSN+1,1)),...

mean(monthlyTrackingError(yMonthSN:end))/std(monthlyTrackingError(yMonthSN:end))];

end

monthlyIR(yMonthSN+1:end,:)=[];

monthlyIR(:,2)=sqrt(12)*monthlyIR(:,2);

dailyTracingError=y(:,end-1)-indexDailyReturn;

dailyIR=zeros(size(y,1)-1,2);

dailyIR(1:end,1)=y(1:end-1,1);

for yRow=1:size(y,1)-1

dailyIR(yRow,2)=mean(dailyTracingError(yRow:end))/std(dailyTracingError(yRow:end));

end

dailyIR(:,2)=dailyIR(:,2)*sqrt(250);

varargout{varout_infoRatio}=struct('daily',dailyIR,'monthly',monthlyIR);

end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 最大回撤maxDrawDown

if nargout>=varout_maxDrawDown+1

yRowNum=size(y,1);

varargout{varout_maxDrawDown}=zeros(yRowNum-1,2); %varargout{varout_infoRatio}

for i=1:yRowNum-1

varargout{varout_maxDrawDown}(i,1)=maxdrawdown(y(i:end,end));

varargout{varout_maxDrawDown}(i,2)=maxdrawdown(refIndex(i:end));

end

end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 市场指数

if nargout>=varout_index+1

marketIndex=idx(raw{1,dateCol},datestr(huicheEndDateNum,'yyyymmdd'),conn);

varargout{varout_index}=marketIndex;

end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 流通市值

if nargout>=varout_matketCap+1

varargout{varout_matketCap}=marketCap(1:marketCapCol,:);

end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% Famma French 三因素回归

if nargout>=varout_FF3+1

varargout{varout_FF3}=FF3(y(:,[1,end-1]),risklessReturn,conn);

end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% xlsresult

if nargout>=varout_xlsResult+1

xlsResult(:,xlsResultCol+1:end)=[];

xlsResult(end-1:end,1)=[{'隔日收益'};{'累计收益'}];

xlsResult(end,2:end)=num2cell(cumprod(cell2mat(xlsResult(end-1,2:end))));

xlsResult=xlsResult';

varargout{varout_xlsResult}=xlsResult;

end

close(conn);

end

function suby=strconnec(subx)

ssubx=subx(1:end-1);

ssubx=strcat('''',ssubx,''',');

tmp=['''',subx{end},''''];

ssubx=[ssubx;tmp];

suby=cell2mat(ssubx');

end

function sy=monthEndDateNum(sDateNum)

%         输入sDateNum是一个date number, 输出sy给出date number所在月最后一天的datenum

sx=datestr(sDateNum,'yyyymm');

endDay=eomday(str2double(sx(1:4)),str2double(sx(5:6)));

sy=datenum([sx,num2str(endDay)],'yyyymmdd');

end

function y=marketCapFun(windIDCell,fromDate,toDate,conn)

% windIDCell一列的cell

% fromDate, toDate是date sting，'yyyymmdd'形式

% y是一个两列的数值矩阵，给出从fromDate到toDate组合的流通市值之和，第一列为日期，第二列为市值

if isempty(windIDCell{1})

tradeDay=fetch(conn,['select f1_1010 from tb_object_1010 where f1_1010 between ''',fromDate,''' and ''',toDate,''' order by f1_1010']);

fprintf('No stock in the portfolio. Market capitalization equals 0.\n');

y=[str2double(tradeDay),zeros(length(tradeDay),1)];

return;

end

uniqueWindID=sortrows(windIDCell);

windID=strconnec(uniqueWindID);

marketCap=fetch(conn,['select f1_5004,f2_5004,f10_5004 from tb_object_5004 where f1_5004 in (',windID,') and (f2_5004 between ''',fromDate,''' and ''',toDate,''')']);

tradeSeriesMatrix=col2matrix(marketCap);

noDataWindID=uniqueWindID(find(1-ismember(uniqueWindID,tradeSeriesMatrix(2:end,1))));

if ~isempty(noDataWindID)

fprintf('Caution: Can not fetch market capitalization data for stock(s) with windID %s from %s to %s. These data are set to 0.',...

cell2mat(strcat(strcat(noDataWindID,',')')),fromDate,toDate);

missed=[noDataWindID,num2cell(zeros(length(noDataWindID),size(tradeSeriesMatrix,2)-1))]; %所有的交易日内都没有数据，流通市值统一设为0

tradeSeriesMatrix=[tradeSeriesMatrix(1,:);srotrows([tradeSeriesMatrix(2:end,:);missed],1)];

end

for tradeSeriesMatrixRow=2:size(tradeSeriesMatrix,1)

for tradeSeriesMatrixCol=3:size(tradeSeriesMatrix,2)

if isnan(tradeSeriesMatrix{tradeSeriesMatrixRow,tradeSeriesMatrixCol})

tradeSeriesMatrix{tradeSeriesMatrixRow,tradeSeriesMatrixCol}=tradeSeriesMatrix{tradeSeriesMatrixRow,tradeSeriesMatrixCol-1};

end

end

end

%     股票在组合建立当日就没有数据，收盘价设为此后最近有数据交易日的收盘价

tradeSeriesMatrixData=cell2mat(tradeSeriesMatrix(2:end,2:end));

for tradeSeriesMatrixDataRow=1:size(tradeSeriesMatrixData,1)

lastNanCol=find(isnan(tradeSeriesMatrixData(tradeSeriesMatrixDataRow,:)),1,'last');

tradeSeriesMatrixData(tradeSeriesMatrixDataRow,1:lastNanCol)=tradeSeriesMatrixData(tradeSeriesMatrixDataRow,lastNanCol+1);

end

y=[str2double(tradeSeriesMatrix(1,2:end))',sum(tradeSeriesMatrixData)'];

end

function y=col2matrix(x)

% x是一个三列的cell，第一列和第二列为指标，第三列为值，如第一列为证券ID，第二列为一段时间之内的交易日，第三列为一只股票在某个交易日的收盘价

% y是一个cell，有unique(x(:,1))+1行和unique(x(:,2))+1列，y(2:end,1)等于unique(x(:,1)且按降序排列,

% y(1,2:end)等于unique(x(:,2))且按降序排列，y(i,j)的值等于y(i,1)所对应的x第一列指标和y(1,j)所对应的x第二列指标给出的x第三列中的值，若x中没有该值，y(i,j)=NaN

keyWindID=unique(x(:,1));

keyTradeDate=unique(x(:,2));

mapWindID=containers.Map(keyWindID,num2cell(1:length(keyWindID)));

mapTradeDate=containers.Map(keyTradeDate,num2cell(1:length(keyTradeDate)));

tradeDataMatrix=nan(length(keyWindID),length(keyTradeDate));

tradeDataMatrix(sub2ind(size(tradeDataMatrix),cell2mat(values(mapWindID,x(:,1))),cell2mat(values(mapTradeDate,x(:,2)))))=cell2mat(x(:,3));

y=nan(length(keyWindID)+1,length(keyTradeDate)+1);

y=num2cell(y);

y(2:end,1)=keyWindID;

y(1,2:end)=keyTradeDate';

y(2:end,2:end)=num2cell(tradeDataMatrix);

end

function y=idx(fromDate,toDate,conn)

% y是一个是一个structure，每个field是一个数值列向量，给出从fromDate到toDate之间所有交易日的市场指数值

% HS300: S12425; 上证综指：1A0001

y.HS300=fch('S12425');

y.SZZZ=fch('1A0001');

function suby=fch(subx)

suby=fetch(conn,['select f2_1425,f7_1425 from tb_object_1425 where f1_1425=''',subx,...

''' and (f2_1425 between ''',fromDate,''' and ''',toDate,''') order by f2_1425']);

suby=[str2double(suby(:,1)),cell2mat(suby(:,2))];

end

end

function yhat=FF3(x,risklessReturn,conn)

% x是一个两列的cell array，第一列为交易日期，第二列为策略隔日收益率，x等于主程序中y(:,[1,end-1])

% yhat是一个两列的矩阵，第一列是日期，第二列为FF3回归的到的累计收益。在由FF3计算隔日收益时，如果由于数据缺失而无法得到某天的收益，那么该日的收益等于x对应当日的真实收益

xorg=x;

totalDateNum=size(x,1);

SMB=zeros(totalDateNum,2);

HML=zeros(totalDateNum,2);

SMB_HML_sampleRatio=20e-2;

eachFetchDateNum=50;

fetchNum=floor(totalDateNum/eachFetchDateNum)+sign(mod(totalDateNum,eachFetchDateNum));

SMB_HML_num=0;

for fetchSN=1:fetchNum

[fetchSN,fetchNum,size(x,1)]

tic

fromDate_YRow=(fetchSN-1)*eachFetchDateNum+1

toDate_YRow=sign(fetchNum-fetchSN)*fetchSN*eachFetchDateNum+(1-sign(fetchNum-fetchSN))*totalDateNum

mktCap_PB=fetch(conn,['select t5.f2_5004,t5.f10_5004,t5.f15_5004,t5.f7_5004 from tb_object_5004 as t5 inner join tb_object_1425 as t1 ',...

'on (t5.f1_5004=t1.f1_1425 and t5.f2_5004=t1.f2_1425) where (t5.f2_5004 between ''',num2str(x(fromDate_YRow,1)),''' and ''',...

num2str(x(toDate_YRow,1)),''') and t1.f11_1425=-1 order by t5.f2_5004']);

if isempty(mktCap_PB)

fprintf('Caution: fetch no data from %d to %d when caculating FF3.\n',x(fromDate_YRow,1),x(toDate_YRow,1));

continue;

end

mktCap_PB=[str2double(mktCap_PB(:,1)),cell2mat(mktCap_PB(:,2:end))];

%     mktCap_PB_Data中会偶尔掺杂nan，去掉这些行

nanRow=prod(mktCap_PB,2);

mktCap_PB(find(isnan(nanRow)),:)=[];

mktCap_PB=sortrows(mktCap_PB,1);

[~,firstRow]=unique(mktCap_PB(:,1),'first');

[~,lastRow]=unique(mktCap_PB(:,1),'last');

[length(firstRow),length(lastRow)]

for dateSN=1:length(firstRow)

currentDateData=mktCap_PB(firstRow(dateSN):lastRow(dateSN),:);

sampleNum=floor(size(currentDateData,1)*SMB_HML_sampleRatio);

currentDateData=sortrows(currentDateData,2);

SMB(SMB_HML_num+1,:)=[currentDateData(1,1),mean(currentDateData(1:sampleNum,end)-currentDateData(end-sampleNum+1:end,end))/100];

currentDateData=sortrows(currentDateData,3);

HML(SMB_HML_num+1,:)=[currentDateData(1,1),mean(currentDateData(1:sampleNum,end)-currentDateData(end-sampleNum+1:end,end))/100];

SMB_HML_num=SMB_HML_num+1;

end

toc

end

SMB=SMB(1:SMB_HML_num,:);

HML=HML(1:SMB_HML_num,:);

indexHS300=fetch(conn,['select f2_1425,f7_1425 from tb_object_1425 where f1_1425=''S12425'' and (f2_1425 between ''',...

num2str(x(1,1)),''' and ''',num2str(x(end,1)),''') order by f2_1425']);

indexHS300=[str2double(indexHS300(:,1)),cell2mat(indexHS300(:,2))];

marketReturn=[1;indexHS300(2:end,2)./indexHS300(1:end-1,2)];

marketReturn=[indexHS300(:,1),marketReturn];

interTradeDay=intersect(intersect(x(:,1),marketReturn(:,1)),SMB(:,1));

SMB=SMB(find(ismember(SMB(:,1),interTradeDay)),:);

HML=HML(find(ismember(HML(:,1),interTradeDay)),:);

marketReturn=marketReturn(find(ismember(marketReturn(:,1),interTradeDay)),:);

x=x(find(ismember(x(:,1),interTradeDay)),:);

X=[ones(length(interTradeDay),1),marketReturn(:,2)-risklessReturn,SMB(:,2),HML(:,2)];

whichStats={'beta','yhat','r','rsquare','adjrsquare','tstat','fstat'};

stats=regstats(x(:,end)-risklessReturn,X(:,2:end),[],whichStats);

% regCoef=stats.beta;

% yhat=X*regCoef+risklessReturn;

yhat=stats.yhat+risklessReturn;

yhat=[interTradeDay,yhat];

xorg(find(ismember(xorg(:,1),interTradeDay)),:)=yhat; % 对于缺乏数据而不能给出FF3回归结果的交易日，当日的FF3收益等于组合的真实收益

yhat=sortrows(xorg,1);

yhat(:,2)=cumprod(yhat(:,2));

yhat=struct('yhat',yhat,'stats',stats);

end