FCI-1 Web Scraping & Visualization

Example: US Treasury Daily Yields

Resource Center | U.S. Department of the Treasury

Official websites use .gov A .gov website belongs to an official government organization in the United States.

https://home.treasury.gov/resource-center/data-chart-center/interest-rates/TextView?type=daily_treasury_yield_curve&field_tdr_date_value=2019

Web Scraping


def obtain_data(year):
    html = urlopen(f'https://home.treasury.gov/resource-center/data-chart-center/interest-rates/TextView?type=daily_treasury_yield_curve&field_tdr_date_value={year}')
    bsyc = BeautifulSoup(html.read(), "lxml")
    table_list = bsyc.findAll('table')
    rows = table_list[0].findAll('tr')
    headers = rows[0].findAll('th')
    daily_yield_curvers = []
    daily_yield_curvers.append([h.contents[0] for h in [headers[0]] + headers[10:13] + headers[14:]])
    for row in rows[1:]:
        row_data = row.findAll('td')
        daily_yield_curvers.append([row_data[0].contents[0].contents[0]] + [float(d.contents[0].strip()) for d in row_data[10:13] + row_data[14:]])
    
    return daily_yield_curvers

Visualization


def plot_yield_curve_3D(df, year):
    df.columns = [1, 2, 3, 6, 12, 24, 36, 60, 84, 120, 240, 360]
    fig, (ax2, ax1) = plt.subplots(1, 2, figsize=(20, 8), subplot_kw={'projection': '3d'}, gridspec_kw={'width_ratios':[3,2.4]}, dpi=200)
    X = np.arange(df.shape[0])
    Y = np.array(df.columns)
    X, Y = np.meshgrid(X, Y)
    Z = df.values.T

    # 3D Surface Plot
    surface = ax1.plot_surface(X, Y, Z, cmap='RdBu')
    plt.colorbar(surface, ax=ax1, shrink=0.5, aspect=5, orientation='vertical')
    ax1.set_zlim(0, 6)
    ax1.set_title('3D Surface Plot')
    ax1.set_xlabel(f"trading days since {year}'s first trading day")
    ax1.set_ylabel('months to maturity')
    ax1.set_zlabel('rate')

    # 3D Wireframe Plot
    wireframe = ax2.plot_wireframe(X, Y, Z, color='skyblue')
    ax2.set_zlim(0, 6)
    ax2.set_title('3D Wireframe Plot')
    ax2.set_xlabel(f"trading days since {year}'s first trading day")
    ax2.set_ylabel('months to maturity')
    ax2.set_zlabel('rate')

    plt.suptitle(f"US Treasury Daily Yields Curve, {year}", fontsize=18)
    fig.tight_layout()
    fig.savefig(f'./daily_yields_curve_{year}-3d.png')
    # plt.show()


def plot_time_series(df, year):
    fig, axes = plt.subplots(1, 2, figsize=(15, 5), dpi=200)
    df.columns = ['1 Mo', '2 Mo', '3 Mo', '6 Mo', '1 Yr', '2 Yr', '3 Yr', '5 Yr', '7 Yr', '10 Yr', '20 Yr', '30 Yr']
    df.plot(ax=axes[0])
    axes[0].set_title(f'Interest Rate Time Series, {year}')
    axes[0].set_ylabel('rate')
    axes[0].legend(loc='upper right')

    df.columns = [1, 2, 3, 6, 12, 24, 36, 60, 84, 120, 240, 360]
    df.iloc[::20].T.plot(ax=axes[1])
    axes[1].set_title(f'{year} Yield Curves, 20 Day Intervals')
    axes[1].set_ylabel('rate')
    axes[1].set_xlabel('months to maturity')
    axes[1].legend(loc='upper right')
    fig.tight_layout()
    fig.savefig(f'./daily_yields_curve_{year}-2d.png')
    # plt.show()

Main Script


if __name__ == "__main__":
    YEAR = 2019
    daily_yield_curvers = obtain_data(YEAR)
    df = pd.DataFrame(daily_yield_curvers[1:], columns=daily_yield_curvers[0]).set_index('Date')
    plot_yield_curve_3D(df, YEAR)
    plot_time_series(df, YEAR)

Results