Primary user: Kiva field partners. As we can read in "How Kiva Works, The Long Version":
** The proposed solution is there to give Field Partners an insight into the risk they run, i.e. not finding (enough) lenders to backfill the loan**
Information on loans were already entered in MongoDB previously. Here we focus on loading the lenders, and the loans_lenders instances.
# Next line commented out because we only want to run this once
#!python src/load_kiva_lenders_to_mongodb.py
Processing lenders file 1/1614 ... done Processing lenders file 2/1614 ... done Processing lenders file 3/1614 ... done Processing lenders file 4/1614 ... done Processing lenders file 5/1614 ... done Processing lenders file 6/1614 ... done Processing lenders file 7/1614 ... done Processing lenders file 8/1614 ... done Processing lenders file 9/1614 ... done Processing lenders file 10/1614 ... done Processing lenders file 11/1614 ... done Processing lenders file 12/1614 ... done Processing lenders file 13/1614 ... done Processing lenders file 14/1614 ... done Processing lenders file 15/1614 ... done Processing lenders file 16/1614 ... done Processing lenders file 17/1614 ... done Processing lenders file 18/1614 ... done Processing lenders file 19/1614 ... done Processing lenders file 20/1614 ... done Processing lenders file 21/1614 ... done Processing lenders file 22/1614 ... done Processing lenders file 23/1614 ... done Processing lenders file 24/1614 ... done Processing lenders file 25/1614 ... done Processing lenders file 26/1614 ... done Processing lenders file 27/1614 ... done Processing lenders file 28/1614 ... done Processing lenders file 29/1614 ... done Processing lenders file 30/1614 ... done Processing lenders file 31/1614 ... done Processing lenders file 32/1614 ... done Processing lenders file 33/1614 ... done Processing lenders file 34/1614 ... done Processing lenders file 35/1614 ... done Processing lenders file 36/1614 ... done Processing lenders file 37/1614 ... done Processing lenders file 38/1614 ... done Processing lenders file 39/1614 ... done Processing lenders file 40/1614 ... done Processing lenders file 41/1614 ... done Processing lenders file 42/1614 ... done Processing lenders file 43/1614 ... done Processing lenders file 44/1614 ... done Processing lenders file 45/1614 ... done Processing lenders file 46/1614 ... done Processing lenders file 47/1614 ... done Processing lenders file 48/1614 ... done Processing lenders file 49/1614 ... done Processing lenders file 50/1614 ... done Processing lenders file 51/1614 ... done Processing lenders file 52/1614 ... done Processing lenders file 53/1614 ... done Processing lenders file 54/1614 ... done Processing lenders file 55/1614 ... done Processing lenders file 56/1614 ... done Processing lenders file 57/1614 ... done Processing lenders file 58/1614 ... done Processing lenders file 59/1614 ... done Processing lenders file 60/1614 ... done Processing lenders file 61/1614 ... done Processing lenders file 62/1614 ... done Processing lenders file 63/1614 ... done Processing lenders file 64/1614 ... done Processing lenders file 65/1614 ... done Processing lenders file 66/1614 ... done Processing lenders file 67/1614 ... done Processing lenders file 68/1614 ... done Processing lenders file 69/1614 ... done Processing lenders file 70/1614 ... done Processing lenders file 71/1614 ... done Processing lenders file 72/1614 ... done Processing lenders file 73/1614 ... done Processing lenders file 74/1614 ... done Processing lenders file 75/1614 ... done Processing lenders file 76/1614 ... done Processing lenders file 77/1614 ... done Processing lenders file 78/1614 ... done Processing lenders file 79/1614 ... done Processing lenders file 80/1614 ... done Processing lenders file 81/1614 ... done Processing lenders file 82/1614 ... done Processing lenders file 83/1614 ... done Processing lenders file 84/1614 ... done Processing lenders file 85/1614 ... done Processing lenders file 86/1614 ... done Processing lenders file 87/1614 ... done Processing lenders file 88/1614 ... done Processing lenders file 89/1614 ... done Processing lenders file 90/1614 ... done Processing lenders file 91/1614 ... done Processing lenders file 92/1614 ... done Processing lenders file 93/1614 ... done Processing lenders file 94/1614 ... done Processing lenders file 95/1614 ... done Processing lenders file 96/1614 ... done Processing lenders file 97/1614 ... done Processing lenders file 98/1614 ... done Processing lenders file 99/1614 ... done Processing lenders file 100/1614 ... done Processing lenders file 101/1614 ... done Processing lenders file 102/1614 ... done Processing lenders file 103/1614 ... done Processing lenders file 104/1614 ... done Processing lenders file 105/1614 ... done Processing lenders file 106/1614 ... done Processing lenders file 107/1614 ... done Processing lenders file 108/1614 ... done Processing lenders file 109/1614 ... done Processing lenders file 110/1614 ... done Processing lenders file 111/1614 ... done Processing lenders file 112/1614 ... done Processing lenders file 113/1614 ... done Processing lenders file 114/1614 ... done Processing lenders file 115/1614 ... done Processing lenders file 116/1614 ... done Processing lenders file 117/1614 ... done Processing lenders file 118/1614 ... done Processing lenders file 119/1614 ... done Processing lenders file 120/1614 ... done Processing lenders file 121/1614 ... done Processing lenders file 122/1614 ... done Processing lenders file 123/1614 ... done Processing lenders file 124/1614 ... done Processing lenders file 125/1614 ... done Processing lenders file 126/1614 ... done Processing lenders file 127/1614 ... done Processing lenders file 128/1614 ... done Processing lenders file 129/1614 ... done Processing lenders file 130/1614 ... done Processing lenders file 131/1614 ... done Processing lenders file 132/1614 ... done Processing lenders file 133/1614 ... done Processing lenders file 134/1614 ... done Processing lenders file 135/1614 ... done Processing lenders file 136/1614 ... done Processing lenders file 137/1614 ... done Processing lenders file 138/1614 ... done Processing lenders file 139/1614 ... done Processing lenders file 140/1614 ... done Processing lenders file 141/1614 ... done Processing lenders file 142/1614 ... done Processing lenders file 143/1614 ... done Processing lenders file 144/1614 ... done Processing lenders file 145/1614 ... done Processing lenders file 146/1614 ... done Processing lenders file 147/1614 ... done Processing lenders file 148/1614 ... done Processing lenders file 149/1614 ... done Processing lenders file 150/1614 ... done Processing lenders file 151/1614 ... done Processing lenders file 152/1614 ... done Processing lenders file 153/1614 ... done Processing lenders file 154/1614 ... done Processing lenders file 155/1614 ... done Processing lenders file 156/1614 ... done Processing lenders file 157/1614 ... done Processing lenders file 158/1614 ... done Processing lenders file 159/1614 ... done Processing lenders file 160/1614 ... done Processing lenders file 161/1614 ... done Processing lenders file 162/1614 ... done Processing lenders file 163/1614 ... done Processing lenders file 164/1614 ... done Processing lenders file 165/1614 ... done Processing lenders file 166/1614 ... done Processing lenders file 167/1614 ... done Processing lenders file 168/1614 ... done Processing lenders file 169/1614 ... done Processing lenders file 170/1614 ... done Processing lenders file 171/1614 ... done Processing lenders file 172/1614 ... done Processing lenders file 173/1614 ... done Processing lenders file 174/1614 ... done Processing lenders file 175/1614 ... done Processing lenders file 176/1614 ... done Processing lenders file 177/1614 ... done Processing lenders file 178/1614 ... done Processing lenders file 179/1614 ... done Processing lenders file 180/1614 ... done Processing lenders file 181/1614 ... done Processing lenders file 182/1614 ... done Processing lenders file 183/1614 ... done Processing lenders file 184/1614 ... done Processing lenders file 185/1614 ... done Processing lenders file 186/1614 ... done Processing lenders file 187/1614 ... done Processing lenders file 188/1614 ... done Processing lenders file 189/1614 ... done Processing lenders file 190/1614 ... done Processing lenders file 191/1614 ... done Processing lenders file 192/1614 ... done Processing lenders file 193/1614 ... done Processing lenders file 194/1614 ... done Processing lenders file 195/1614 ... done Processing lenders file 196/1614 ... done Processing lenders file 197/1614 ... done Processing lenders file 198/1614 ... done Processing lenders file 199/1614 ... done Processing lenders file 200/1614 ... done Processing lenders file 201/1614 ... done Processing lenders file 202/1614 ... done Processing lenders file 203/1614 ... done Processing lenders file 204/1614 ... done Processing lenders file 205/1614 ... done Processing lenders file 206/1614 ... done Processing lenders file 207/1614 ... done Processing lenders file 208/1614 ... done Processing lenders file 209/1614 ... done Processing lenders file 210/1614 ... done Processing lenders file 211/1614 ... done Processing lenders file 212/1614 ... done Processing lenders file 213/1614 ... done Processing lenders file 214/1614 ... done Processing lenders file 215/1614 ... done Processing lenders file 216/1614 ... done Processing lenders file 217/1614 ... done Processing lenders file 218/1614 ... done Processing lenders file 219/1614 ... done Processing lenders file 220/1614 ... done Processing lenders file 221/1614 ... done Processing lenders file 222/1614 ... done Processing lenders file 223/1614 ... done Processing lenders file 224/1614 ... done Processing lenders file 225/1614 ... done Processing lenders file 226/1614 ... done Processing lenders file 227/1614 ... done Processing lenders file 228/1614 ... done Processing lenders file 229/1614 ... done Processing lenders file 230/1614 ... done Processing lenders file 231/1614 ... done Processing lenders file 232/1614 ... done Processing lenders file 233/1614 ... done Processing lenders file 234/1614 ... done Processing lenders file 235/1614 ... done Processing lenders file 236/1614 ... done Processing lenders file 237/1614 ... done Processing lenders file 238/1614 ... done Processing lenders file 239/1614 ... done Processing lenders file 240/1614 ... done Processing lenders file 241/1614 ... done Processing lenders file 242/1614 ... done Processing lenders file 243/1614 ... done Processing lenders file 244/1614 ... done Processing lenders file 245/1614 ... done Processing lenders file 246/1614 ... done Processing lenders file 247/1614 ... done Processing lenders file 248/1614 ... done Processing lenders file 249/1614 ... done Processing lenders file 250/1614 ... done Processing lenders file 251/1614 ... done Processing lenders file 252/1614 ... done Processing lenders file 253/1614 ... done Processing lenders file 254/1614 ... done Processing lenders file 255/1614 ... done Processing lenders file 256/1614 ... done Processing lenders file 257/1614 ... done Processing lenders file 258/1614 ... done Processing lenders file 259/1614 ... done Processing lenders file 260/1614 ... done Processing lenders file 261/1614 ... done Processing lenders file 262/1614 ... done Processing lenders file 263/1614 ... done Processing lenders file 264/1614 ... done Processing lenders file 265/1614 ... done Processing lenders file 266/1614 ... done Processing lenders file 267/1614 ... done Processing lenders file 268/1614 ... done Processing lenders file 269/1614 ... done Processing lenders file 270/1614 ... done Processing lenders file 271/1614 ... done Processing lenders file 272/1614 ... done Processing lenders file 273/1614 ... done Processing lenders file 274/1614 ... done Processing lenders file 275/1614 ... done Processing lenders file 276/1614 ... done Processing lenders file 277/1614 ... done Processing lenders file 278/1614 ... done Processing lenders file 279/1614 ... done Processing lenders file 280/1614 ... done Processing lenders file 281/1614 ... done Processing lenders file 282/1614 ... done Processing lenders file 283/1614 ... done Processing lenders file 284/1614 ... done Processing lenders file 285/1614 ... done Processing lenders file 286/1614 ... done Processing lenders file 287/1614 ... done Processing lenders file 288/1614 ... done Processing lenders file 289/1614 ... done Processing lenders file 290/1614 ... done Processing lenders file 291/1614 ... done Processing lenders file 292/1614 ... done Processing lenders file 293/1614 ... done Processing lenders file 294/1614 ... done Processing lenders file 295/1614 ... done Processing lenders file 296/1614 ... done Processing lenders file 297/1614 ... done Processing lenders file 298/1614 ... done Processing lenders file 299/1614 ... done Processing lenders file 300/1614 ... done Processing lenders file 301/1614 ... done Processing lenders file 302/1614 ... done Processing lenders file 303/1614 ... done Processing lenders file 304/1614 ... done Processing lenders file 305/1614 ... done Processing lenders file 306/1614 ... done Processing lenders file 307/1614 ... done Processing lenders file 308/1614 ... done Processing lenders file 309/1614 ... done Processing lenders file 310/1614 ... done Processing lenders file 311/1614 ... done Processing lenders file 312/1614 ... done Processing lenders file 313/1614 ... done Processing lenders file 314/1614 ... done Processing lenders file 315/1614 ... done Processing lenders file 316/1614 ... done Processing lenders file 317/1614 ... done Processing lenders file 318/1614 ... done Processing lenders file 319/1614 ... done Processing lenders file 320/1614 ... done Processing lenders file 321/1614 ... done Processing lenders file 322/1614 ... done Processing lenders file 323/1614 ... done Processing lenders file 324/1614 ... done Processing lenders file 325/1614 ... done Processing lenders file 326/1614 ... done Processing lenders file 327/1614 ... done Processing lenders file 328/1614 ... done Processing lenders file 329/1614 ... done Processing lenders file 330/1614 ... done Processing lenders file 331/1614 ... done Processing lenders file 332/1614 ... done Processing lenders file 333/1614 ... done Processing lenders file 334/1614 ... done Processing lenders file 335/1614 ... done Processing lenders file 336/1614 ... done Processing lenders file 337/1614 ... done Processing lenders file 338/1614 ... done Processing lenders file 339/1614 ... done Processing lenders file 340/1614 ... done Processing lenders file 341/1614 ... done Processing lenders file 342/1614 ... done Processing lenders file 343/1614 ... done Processing lenders file 344/1614 ... done Processing lenders file 345/1614 ... done Processing lenders file 346/1614 ... done Processing lenders file 347/1614 ... done Processing lenders file 348/1614 ... done Processing lenders file 349/1614 ... done Processing lenders file 350/1614 ... done Processing lenders file 351/1614 ... done Processing lenders file 352/1614 ... done Processing lenders file 353/1614 ... done Processing lenders file 354/1614 ... done Processing lenders file 355/1614 ... done Processing lenders file 356/1614 ... done Processing lenders file 357/1614 ... done Processing lenders file 358/1614 ... done Processing lenders file 359/1614 ... done Processing lenders file 360/1614 ... done Processing lenders file 361/1614 ... done Processing lenders file 362/1614 ... done Processing lenders file 363/1614 ... done Processing lenders file 364/1614 ... done Processing lenders file 365/1614 ... done Processing lenders file 366/1614 ... done Processing lenders file 367/1614 ... done Processing lenders file 368/1614 ... done Processing lenders file 369/1614 ... done Processing lenders file 370/1614 ... done Processing lenders file 371/1614 ... done Processing lenders file 372/1614 ... done Processing lenders file 373/1614 ... done Processing lenders file 374/1614 ... done Processing lenders file 375/1614 ... done Processing lenders file 376/1614 ... done Processing lenders file 377/1614 ... done Processing lenders file 378/1614 ... done Processing lenders file 379/1614 ... done Processing lenders file 380/1614 ... done Processing lenders file 381/1614 ... done Processing lenders file 382/1614 ... done Processing lenders file 383/1614 ... done Processing lenders file 384/1614 ... done Processing lenders file 385/1614 ... done Processing lenders file 386/1614 ... done Processing lenders file 387/1614 ... done Processing lenders file 388/1614 ... done Processing lenders file 389/1614 ... done Processing lenders file 390/1614 ... done Processing lenders file 391/1614 ... done Processing lenders file 392/1614 ... done Processing lenders file 393/1614 ... done Processing lenders file 394/1614 ... done Processing lenders file 395/1614 ... done Processing lenders file 396/1614 ... done Processing lenders file 397/1614 ... done Processing lenders file 398/1614 ... done Processing lenders file 399/1614 ... done Processing lenders file 400/1614 ... done Processing lenders file 401/1614 ... done Processing lenders file 402/1614 ... done Processing lenders file 403/1614 ... done Processing lenders file 404/1614 ... done Processing lenders file 405/1614 ... done Processing lenders file 406/1614 ... done Processing lenders file 407/1614 ... done Processing lenders file 408/1614 ... done Processing lenders file 409/1614 ... done Processing lenders file 410/1614 ... done Processing lenders file 411/1614 ... done Processing lenders file 412/1614 ... done Processing lenders file 413/1614 ... done Processing lenders file 414/1614 ... done Processing lenders file 415/1614 ... done Processing lenders file 416/1614 ... done Processing lenders file 417/1614 ... done Processing lenders file 418/1614 ... done Processing lenders file 419/1614 ... done Processing lenders file 420/1614 ... done Processing lenders file 421/1614 ... done Processing lenders file 422/1614 ... done Processing lenders file 423/1614 ... done Processing lenders file 424/1614 ... done Processing lenders file 425/1614 ... done Processing lenders file 426/1614 ... done Processing lenders file 427/1614 ... done Processing lenders file 428/1614 ... done Processing lenders file 429/1614 ... done Processing lenders file 430/1614 ... done Processing lenders file 431/1614 ... done Processing lenders file 432/1614 ... done Processing lenders file 433/1614 ... done Processing lenders file 434/1614 ... done Processing lenders file 435/1614 ... done Processing lenders file 436/1614 ... done Processing lenders file 437/1614 ... done Processing lenders file 438/1614 ... done Processing lenders file 439/1614 ... done Processing lenders file 440/1614 ... done Processing lenders file 441/1614 ... done Processing lenders file 442/1614 ... done Processing lenders file 443/1614 ... done Processing lenders file 444/1614 ... done Processing lenders file 445/1614 ... done Processing lenders file 446/1614 ... done Processing lenders file 447/1614 ... done Processing lenders file 448/1614 ... done Processing lenders file 449/1614 ... done Processing lenders file 450/1614 ... done Processing lenders file 451/1614 ... done Processing lenders file 452/1614 ... done Processing lenders file 453/1614 ... done Processing lenders file 454/1614 ... done Processing lenders file 455/1614 ... done Processing lenders file 456/1614 ... done Processing lenders file 457/1614 ... done Processing lenders file 458/1614 ... done Processing lenders file 459/1614 ... done Processing lenders file 460/1614 ... done Processing lenders file 461/1614 ... done Processing lenders file 462/1614 ... done Processing lenders file 463/1614 ... done Processing lenders file 464/1614 ... done Processing lenders file 465/1614 ... done Processing lenders file 466/1614 ... done Processing lenders file 467/1614 ... done Processing lenders file 468/1614 ... done Processing lenders file 469/1614 ... done Processing lenders file 470/1614 ... done Processing lenders file 471/1614 ... done Processing lenders file 472/1614 ... done Processing lenders file 473/1614 ... done Processing lenders file 474/1614 ... done Processing lenders file 475/1614 ... done Processing lenders file 476/1614 ... done Processing lenders file 477/1614 ... done Processing lenders file 478/1614 ... done Processing lenders file 479/1614 ... done Processing lenders file 480/1614 ... done Processing lenders file 481/1614 ... done Processing lenders file 482/1614 ... done Processing lenders file 483/1614 ... done Processing lenders file 484/1614 ... done Processing lenders file 485/1614 ... done Processing lenders file 486/1614 ... done Processing lenders file 487/1614 ... done Processing lenders file 488/1614 ... done Processing lenders file 489/1614 ... done Processing lenders file 490/1614 ... done Processing lenders file 491/1614 ... done Processing lenders file 492/1614 ... done Processing lenders file 493/1614 ... done Processing lenders file 494/1614 ... done Processing lenders file 495/1614 ... done Processing lenders file 496/1614 ... done Processing lenders file 497/1614 ... done Processing lenders file 498/1614 ... done Processing lenders file 499/1614 ... done Processing lenders file 500/1614 ... done Processing lenders file 501/1614 ... done Processing lenders file 502/1614 ... done Processing lenders file 503/1614 ... done Processing lenders file 504/1614 ... done Processing lenders file 505/1614 ... done Processing lenders file 506/1614 ... done Processing lenders file 507/1614 ... done Processing lenders file 508/1614 ... done Processing lenders file 509/1614 ... done Processing lenders file 510/1614 ... done Processing lenders file 511/1614 ... done Processing lenders file 512/1614 ... done Processing lenders file 513/1614 ... done Processing lenders file 514/1614 ... done Processing lenders file 515/1614 ... done Processing lenders file 516/1614 ... done Processing lenders file 517/1614 ... done Processing lenders file 518/1614 ... done Processing lenders file 519/1614 ... done Processing lenders file 520/1614 ... done Processing lenders file 521/1614 ... done Processing lenders file 522/1614 ... done Processing lenders file 523/1614 ... done Processing lenders file 524/1614 ... done Processing lenders file 525/1614 ... done Processing lenders file 526/1614 ... done Processing lenders file 527/1614 ... done Processing lenders file 528/1614 ... done Processing lenders file 529/1614 ... done Processing lenders file 530/1614 ... done Processing lenders file 531/1614 ... done Processing lenders file 532/1614 ... done Processing lenders file 533/1614 ... done Processing lenders file 534/1614 ... done Processing lenders file 535/1614 ... done Processing lenders file 536/1614 ... done Processing lenders file 537/1614 ... done Processing lenders file 538/1614 ... done Processing lenders file 539/1614 ... done Processing lenders file 540/1614 ... done Processing lenders file 541/1614 ... done Processing lenders file 542/1614 ... done Processing lenders file 543/1614 ... done Processing lenders file 544/1614 ... done Processing lenders file 545/1614 ... done Processing lenders file 546/1614 ... done Processing lenders file 547/1614 ... done Processing lenders file 548/1614 ... done Processing lenders file 549/1614 ... done Processing lenders file 550/1614 ... done Processing lenders file 551/1614 ... done Processing lenders file 552/1614 ... done Processing lenders file 553/1614 ... done Processing lenders file 554/1614 ... done Processing lenders file 555/1614 ... done Processing lenders file 556/1614 ... done Processing lenders file 557/1614 ... done Processing lenders file 558/1614 ... done Processing lenders file 559/1614 ... done Processing lenders file 560/1614 ... done Processing lenders file 561/1614 ... done Processing lenders file 562/1614 ... done Processing lenders file 563/1614 ... done Processing lenders file 564/1614 ... done Processing lenders file 565/1614 ... done Processing lenders file 566/1614 ... done Processing lenders file 567/1614 ... done Processing lenders file 568/1614 ... done Processing lenders file 569/1614 ... done Processing lenders file 570/1614 ... done Processing lenders file 571/1614 ... done Processing lenders file 572/1614 ... done Processing lenders file 573/1614 ... done Processing lenders file 574/1614 ... done Processing lenders file 575/1614 ... done Processing lenders file 576/1614 ... done Processing lenders file 577/1614 ... done Processing lenders file 578/1614 ... done Processing lenders file 579/1614 ... done Processing lenders file 580/1614 ... done Processing lenders file 581/1614 ... done Processing lenders file 582/1614 ... done Processing lenders file 583/1614 ... done Processing lenders file 584/1614 ... done Processing lenders file 585/1614 ... done Processing lenders file 586/1614 ... done Processing lenders file 587/1614 ... done Processing lenders file 588/1614 ... done Processing lenders file 589/1614 ... done Processing lenders file 590/1614 ... done Processing lenders file 591/1614 ... done Processing lenders file 592/1614 ... done Processing lenders file 593/1614 ... done Processing lenders file 594/1614 ... done Processing lenders file 595/1614 ... done Processing lenders file 596/1614 ... done Processing lenders file 597/1614 ... done Processing lenders file 598/1614 ... done Processing lenders file 599/1614 ... done Processing lenders file 600/1614 ... done Processing lenders file 601/1614 ... done Processing lenders file 602/1614 ... done Processing lenders file 603/1614 ... done Processing lenders file 604/1614 ... done Processing lenders file 605/1614 ... done Processing lenders file 606/1614 ... done Processing lenders file 607/1614 ... done Processing lenders file 608/1614 ... done Processing lenders file 609/1614 ... done Processing lenders file 610/1614 ... done Processing lenders file 611/1614 ... done Processing lenders file 612/1614 ... done Processing lenders file 613/1614 ... done Processing lenders file 614/1614 ... done Processing lenders file 615/1614 ... done Processing lenders file 616/1614 ... done Processing lenders file 617/1614 ... done Processing lenders file 618/1614 ... done Processing lenders file 619/1614 ... done Processing lenders file 620/1614 ... done Processing lenders file 621/1614 ... done Processing lenders file 622/1614 ... done Processing lenders file 623/1614 ... done Processing lenders file 624/1614 ... done Processing lenders file 625/1614 ... done Processing lenders file 626/1614 ... done Processing lenders file 627/1614 ... done Processing lenders file 628/1614 ... done Processing lenders file 629/1614 ... done Processing lenders file 630/1614 ... done Processing lenders file 631/1614 ... done Processing lenders file 632/1614 ... done Processing lenders file 633/1614 ... done Processing lenders file 634/1614 ... done Processing lenders file 635/1614 ... done Processing lenders file 636/1614 ... done Processing lenders file 637/1614 ... done Processing lenders file 638/1614 ... done Processing lenders file 639/1614 ... done Processing lenders file 640/1614 ... done Processing lenders file 641/1614 ... done Processing lenders file 642/1614 ... done Processing lenders file 643/1614 ... done Processing lenders file 644/1614 ... done Processing lenders file 645/1614 ... done Processing lenders file 646/1614 ... done Processing lenders file 647/1614 ... done Processing lenders file 648/1614 ... done Processing lenders file 649/1614 ... done Processing lenders file 650/1614 ... done Processing lenders file 651/1614 ... done Processing lenders file 652/1614 ... done Processing lenders file 653/1614 ... done Processing lenders file 654/1614 ... done Processing lenders file 655/1614 ... done Processing lenders file 656/1614 ... done Processing lenders file 657/1614 ... done Processing lenders file 658/1614 ... done Processing lenders file 659/1614 ... done Processing lenders file 660/1614 ... done Processing lenders file 661/1614 ... done Processing lenders file 662/1614 ... done Processing lenders file 663/1614 ... done Processing lenders file 664/1614 ... done Processing lenders file 665/1614 ... done Processing lenders file 666/1614 ... done Processing lenders file 667/1614 ... done Processing lenders file 668/1614 ... done Processing lenders file 669/1614 ... done Processing lenders file 670/1614 ... done Processing lenders file 671/1614 ... done Processing lenders file 672/1614 ... done Processing lenders file 673/1614 ... done Processing lenders file 674/1614 ... done Processing lenders file 675/1614 ... done Processing lenders file 676/1614 ... done Processing lenders file 677/1614 ... done Processing lenders file 678/1614 ... done Processing lenders file 679/1614 ... done Processing lenders file 680/1614 ... done Processing lenders file 681/1614 ... done Processing lenders file 682/1614 ... done Processing lenders file 683/1614 ... done Processing lenders file 684/1614 ... done Processing lenders file 685/1614 ... done Processing lenders file 686/1614 ... done Processing lenders file 687/1614 ... done Processing lenders file 688/1614 ... done Processing lenders file 689/1614 ... done Processing lenders file 690/1614 ... done Processing lenders file 691/1614 ... done Processing lenders file 692/1614 ... done Processing lenders file 693/1614 ... done Processing lenders file 694/1614 ... done Processing lenders file 695/1614 ... done Processing lenders file 696/1614 ... done Processing lenders file 697/1614 ... done Processing lenders file 698/1614 ... done Processing lenders file 699/1614 ... done Processing lenders file 700/1614 ... done Processing lenders file 701/1614 ... done Processing lenders file 702/1614 ... done Processing lenders file 703/1614 ... done Processing lenders file 704/1614 ... done Processing lenders file 705/1614 ... done Processing lenders file 706/1614 ... done Processing lenders file 707/1614 ... done Processing lenders file 708/1614 ... done Processing lenders file 709/1614 ... done Processing lenders file 710/1614 ... done Processing lenders file 711/1614 ... done Processing lenders file 712/1614 ... done Processing lenders file 713/1614 ... done Processing lenders file 714/1614 ... done Processing lenders file 715/1614 ... done Processing lenders file 716/1614 ... done Processing lenders file 717/1614 ... done Processing lenders file 718/1614 ... done Processing lenders file 719/1614 ... done Processing lenders file 720/1614 ... done Processing lenders file 721/1614 ... done Processing lenders file 722/1614 ... done Processing lenders file 723/1614 ... done Processing lenders file 724/1614 ... done Processing lenders file 725/1614 ... done Processing lenders file 726/1614 ... done Processing lenders file 727/1614 ... done Processing lenders file 728/1614 ... done Processing lenders file 729/1614 ... done Processing lenders file 730/1614 ... done Processing lenders file 731/1614 ... done Processing lenders file 732/1614 ... done Processing lenders file 733/1614 ... done Processing lenders file 734/1614 ... done Processing lenders file 735/1614 ... done Processing lenders file 736/1614 ... done Processing lenders file 737/1614 ... done Processing lenders file 738/1614 ... done Processing lenders file 739/1614 ... done Processing lenders file 740/1614 ... done Processing lenders file 741/1614 ... done Processing lenders file 742/1614 ... done Processing lenders file 743/1614 ... done Processing lenders file 744/1614 ... done Processing lenders file 745/1614 ... done Processing lenders file 746/1614 ... done Processing lenders file 747/1614 ... done Processing lenders file 748/1614 ... done Processing lenders file 749/1614 ... done Processing lenders file 750/1614 ... done Processing lenders file 751/1614 ... done Processing lenders file 752/1614 ... done Processing lenders file 753/1614 ... done Processing lenders file 754/1614 ... done Processing lenders file 755/1614 ... done Processing lenders file 756/1614 ... done Processing lenders file 757/1614 ... done Processing lenders file 758/1614 ... done Processing lenders file 759/1614 ... done Processing lenders file 760/1614 ... done Processing lenders file 761/1614 ... done Processing lenders file 762/1614 ... done Processing lenders file 763/1614 ... done Processing lenders file 764/1614 ... done Processing lenders file 765/1614 ... done Processing lenders file 766/1614 ... done Processing lenders file 767/1614 ... done Processing lenders file 768/1614 ... done Processing lenders file 769/1614 ... done Processing lenders file 770/1614 ... done Processing lenders file 771/1614 ... done Processing lenders file 772/1614 ... done Processing lenders file 773/1614 ... done Processing lenders file 774/1614 ... done Processing lenders file 775/1614 ... done Processing lenders file 776/1614 ... done Processing lenders file 777/1614 ... done Processing lenders file 778/1614 ... done Processing lenders file 779/1614 ... done Processing lenders file 780/1614 ... done Processing lenders file 781/1614 ... done Processing lenders file 782/1614 ... done Processing lenders file 783/1614 ... done Processing lenders file 784/1614 ... done Processing lenders file 785/1614 ... done Processing lenders file 786/1614 ... done Processing lenders file 787/1614 ... done Processing lenders file 788/1614 ... done Processing lenders file 789/1614 ... done Processing lenders file 790/1614 ... done Processing lenders file 791/1614 ... done Processing lenders file 792/1614 ... done Processing lenders file 793/1614 ... done Processing lenders file 794/1614 ... done Processing lenders file 795/1614 ... done Processing lenders file 796/1614 ... done Processing lenders file 797/1614 ... done Processing lenders file 798/1614 ... done Processing lenders file 799/1614 ... done Processing lenders file 800/1614 ... done Processing lenders file 801/1614 ... done Processing lenders file 802/1614 ... done Processing lenders file 803/1614 ... done Processing lenders file 804/1614 ... done Processing lenders file 805/1614 ... done Processing lenders file 806/1614 ... done Processing lenders file 807/1614 ... done Processing lenders file 808/1614 ... done Processing lenders file 809/1614 ... done Processing lenders file 810/1614 ... done Processing lenders file 811/1614 ... done Processing lenders file 812/1614 ... done Processing lenders file 813/1614 ... done Processing lenders file 814/1614 ... done Processing lenders file 815/1614 ... done Processing lenders file 816/1614 ... done Processing lenders file 817/1614 ... done Processing lenders file 818/1614 ... done Processing lenders file 819/1614 ... done Processing lenders file 820/1614 ... done Processing lenders file 821/1614 ... done Processing lenders file 822/1614 ... done Processing lenders file 823/1614 ... done Processing lenders file 824/1614 ... done Processing lenders file 825/1614 ... done Processing lenders file 826/1614 ... done Processing lenders file 827/1614 ... done Processing lenders file 828/1614 ... done Processing lenders file 829/1614 ... done Processing lenders file 830/1614 ... done Processing lenders file 831/1614 ... done Processing lenders file 832/1614 ... done Processing lenders file 833/1614 ... done Processing lenders file 834/1614 ... done Processing lenders file 835/1614 ... done Processing lenders file 836/1614 ... done Processing lenders file 837/1614 ... done Processing lenders file 838/1614 ... done Processing lenders file 839/1614 ... done Processing lenders file 840/1614 ... done Processing lenders file 841/1614 ... done Processing lenders file 842/1614 ... done Processing lenders file 843/1614 ... done Processing lenders file 844/1614 ... done Processing lenders file 845/1614 ... done Processing lenders file 846/1614 ... done Processing lenders file 847/1614 ... done Processing lenders file 848/1614 ... done Processing lenders file 849/1614 ... done Processing lenders file 850/1614 ... done Processing lenders file 851/1614 ... done Processing lenders file 852/1614 ... done Processing lenders file 853/1614 ... done Processing lenders file 854/1614 ... done Processing lenders file 855/1614 ... done Processing lenders file 856/1614 ... done Processing lenders file 857/1614 ... done Processing lenders file 858/1614 ... done Processing lenders file 859/1614 ... done Processing lenders file 860/1614 ... done Processing lenders file 861/1614 ... done Processing lenders file 862/1614 ... done Processing lenders file 863/1614 ... done Processing lenders file 864/1614 ... done Processing lenders file 865/1614 ... done Processing lenders file 866/1614 ... done Processing lenders file 867/1614 ... done Processing lenders file 868/1614 ... done Processing lenders file 869/1614 ... done Processing lenders file 870/1614 ... done Processing lenders file 871/1614 ... done Processing lenders file 872/1614 ... done Processing lenders file 873/1614 ... done Processing lenders file 874/1614 ... done Processing lenders file 875/1614 ... done Processing lenders file 876/1614 ... done Processing lenders file 877/1614 ... done Processing lenders file 878/1614 ... done Processing lenders file 879/1614 ... done Processing lenders file 880/1614 ... done Processing lenders file 881/1614 ... done Processing lenders file 882/1614 ... done Processing lenders file 883/1614 ... done Processing lenders file 884/1614 ... done Processing lenders file 885/1614 ... done Processing lenders file 886/1614 ... done Processing lenders file 887/1614 ... done Processing lenders file 888/1614 ... done Processing lenders file 889/1614 ... done Processing lenders file 890/1614 ... done Processing lenders file 891/1614 ... done Processing lenders file 892/1614 ... done Processing lenders file 893/1614 ... done Processing lenders file 894/1614 ... done Processing lenders file 895/1614 ... done Processing lenders file 896/1614 ... done Processing lenders file 897/1614 ... done Processing lenders file 898/1614 ... done Processing lenders file 899/1614 ... done Processing lenders file 900/1614 ... done Processing lenders file 901/1614 ... done Processing lenders file 902/1614 ... done Processing lenders file 903/1614 ... done Processing lenders file 904/1614 ... done Processing lenders file 905/1614 ... done Processing lenders file 906/1614 ... done Processing lenders file 907/1614 ... done Processing lenders file 908/1614 ... done Processing lenders file 909/1614 ... done Processing lenders file 910/1614 ... done Processing lenders file 911/1614 ... done Processing lenders file 912/1614 ... done Processing lenders file 913/1614 ... done Processing lenders file 914/1614 ... done Processing lenders file 915/1614 ... done Processing lenders file 916/1614 ... done Processing lenders file 917/1614 ... done Processing lenders file 918/1614 ... done Processing lenders file 919/1614 ... done Processing lenders file 920/1614 ... done Processing lenders file 921/1614 ... done Processing lenders file 922/1614 ... done Processing lenders file 923/1614 ... done Processing lenders file 924/1614 ... done Processing lenders file 925/1614 ... done Processing lenders file 926/1614 ... done Processing lenders file 927/1614 ... done Processing lenders file 928/1614 ... done Processing lenders file 929/1614 ... done Processing lenders file 930/1614 ... done Processing lenders file 931/1614 ... done Processing lenders file 932/1614 ... done Processing lenders file 933/1614 ... done Processing lenders file 934/1614 ... done Processing lenders file 935/1614 ... done Processing lenders file 936/1614 ... done Processing lenders file 937/1614 ... done Processing lenders file 938/1614 ... done Processing lenders file 939/1614 ... done Processing lenders file 940/1614 ... done Processing lenders file 941/1614 ... done Processing lenders file 942/1614 ... done Processing lenders file 943/1614 ... done Processing lenders file 944/1614 ... done Processing lenders file 945/1614 ... done Processing lenders file 946/1614 ... done Processing lenders file 947/1614 ... done Processing lenders file 948/1614 ... done Processing lenders file 949/1614 ... done Processing lenders file 950/1614 ... done Processing lenders file 951/1614 ... done Processing lenders file 952/1614 ... done Processing lenders file 953/1614 ... done Processing lenders file 954/1614 ... done Processing lenders file 955/1614 ... done Processing lenders file 956/1614 ... done Processing lenders file 957/1614 ... done Processing lenders file 958/1614 ... done Processing lenders file 959/1614 ... done Processing lenders file 960/1614 ... done Processing lenders file 961/1614 ... done Processing lenders file 962/1614 ... done Processing lenders file 963/1614 ... done Processing lenders file 964/1614 ... done Processing lenders file 965/1614 ... done Processing lenders file 966/1614 ... done Processing lenders file 967/1614 ... done Processing lenders file 968/1614 ... done Processing lenders file 969/1614 ... done Processing lenders file 970/1614 ... done Processing lenders file 971/1614 ... done Processing lenders file 972/1614 ... done Processing lenders file 973/1614 ... done Processing lenders file 974/1614 ... done Processing lenders file 975/1614 ... done Processing lenders file 976/1614 ... done Processing lenders file 977/1614 ... done Processing lenders file 978/1614 ... done Processing lenders file 979/1614 ... done Processing lenders file 980/1614 ... done Processing lenders file 981/1614 ... done Processing lenders file 982/1614 ... done Processing lenders file 983/1614 ... done Processing lenders file 984/1614 ... done Processing lenders file 985/1614 ... done Processing lenders file 986/1614 ... done Processing lenders file 987/1614 ... done Processing lenders file 988/1614 ... done Processing lenders file 989/1614 ... done Processing lenders file 990/1614 ... done Processing lenders file 991/1614 ... done Processing lenders file 992/1614 ... done Processing lenders file 993/1614 ... done Processing lenders file 994/1614 ... done Processing lenders file 995/1614 ... done Processing lenders file 996/1614 ... done Processing lenders file 997/1614 ... done Processing lenders file 998/1614 ... done Processing lenders file 999/1614 ... done Processing lenders file 1000/1614 ... done Processing lenders file 1001/1614 ... done Processing lenders file 1002/1614 ... done Processing lenders file 1003/1614 ... done Processing lenders file 1004/1614 ... done Processing lenders file 1005/1614 ... done Processing lenders file 1006/1614 ... done Processing lenders file 1007/1614 ... done Processing lenders file 1008/1614 ... done Processing lenders file 1009/1614 ... done Processing lenders file 1010/1614 ... done Processing lenders file 1011/1614 ... done Processing lenders file 1012/1614 ... done Processing lenders file 1013/1614 ... done Processing lenders file 1014/1614 ... done Processing lenders file 1015/1614 ... done Processing lenders file 1016/1614 ... done Processing lenders file 1017/1614 ... done Processing lenders file 1018/1614 ... done Processing lenders file 1019/1614 ... done Processing lenders file 1020/1614 ... done Processing lenders file 1021/1614 ... done Processing lenders file 1022/1614 ... done Processing lenders file 1023/1614 ... done Processing lenders file 1024/1614 ... done Processing lenders file 1025/1614 ... done Processing lenders file 1026/1614 ... done Processing lenders file 1027/1614 ... done Processing lenders file 1028/1614 ... done Processing lenders file 1029/1614 ... done Processing lenders file 1030/1614 ... done Processing lenders file 1031/1614 ... done Processing lenders file 1032/1614 ... done Processing lenders file 1033/1614 ... done Processing lenders file 1034/1614 ... done Processing lenders file 1035/1614 ... done Processing lenders file 1036/1614 ... done Processing lenders file 1037/1614 ... done Processing lenders file 1038/1614 ... done Processing lenders file 1039/1614 ... done Processing lenders file 1040/1614 ... done Processing lenders file 1041/1614 ... done Processing lenders file 1042/1614 ... done Processing lenders file 1043/1614 ... done Processing lenders file 1044/1614 ... done Processing lenders file 1045/1614 ... done Processing lenders file 1046/1614 ... done Processing lenders file 1047/1614 ... done Processing lenders file 1048/1614 ... done Processing lenders file 1049/1614 ... done Processing lenders file 1050/1614 ... done Processing lenders file 1051/1614 ... done Processing lenders file 1052/1614 ... done Processing lenders file 1053/1614 ... done Processing lenders file 1054/1614 ... done Processing lenders file 1055/1614 ... done Processing lenders file 1056/1614 ... done Processing lenders file 1057/1614 ... done Processing lenders file 1058/1614 ... done Processing lenders file 1059/1614 ... done Processing lenders file 1060/1614 ... done Processing lenders file 1061/1614 ... done Processing lenders file 1062/1614 ... done Processing lenders file 1063/1614 ... done Processing lenders file 1064/1614 ... done Processing lenders file 1065/1614 ... done Processing lenders file 1066/1614 ... done Processing lenders file 1067/1614 ... done Processing lenders file 1068/1614 ... done Processing lenders file 1069/1614 ... done Processing lenders file 1070/1614 ... done Processing lenders file 1071/1614 ... done Processing lenders file 1072/1614 ... done Processing lenders file 1073/1614 ... done Processing lenders file 1074/1614 ... done Processing lenders file 1075/1614 ... done Processing lenders file 1076/1614 ... done Processing lenders file 1077/1614 ... done Processing lenders file 1078/1614 ... done Processing lenders file 1079/1614 ... done Processing lenders file 1080/1614 ... done Processing lenders file 1081/1614 ... done Processing lenders file 1082/1614 ... done Processing lenders file 1083/1614 ... done Processing lenders file 1084/1614 ... done Processing lenders file 1085/1614 ... done Processing lenders file 1086/1614 ... done Processing lenders file 1087/1614 ... done Processing lenders file 1088/1614 ... done Processing lenders file 1089/1614 ... done Processing lenders file 1090/1614 ... done Processing lenders file 1091/1614 ... done Processing lenders file 1092/1614 ... done Processing lenders file 1093/1614 ... done Processing lenders file 1094/1614 ... done Processing lenders file 1095/1614 ... done Processing lenders file 1096/1614 ... done Processing lenders file 1097/1614 ... done Processing lenders file 1098/1614 ... done Processing lenders file 1099/1614 ... done Processing lenders file 1100/1614 ... done Processing lenders file 1101/1614 ... done Processing lenders file 1102/1614 ... done Processing lenders file 1103/1614 ... done Processing lenders file 1104/1614 ... done Processing lenders file 1105/1614 ... done Processing lenders file 1106/1614 ... done Processing lenders file 1107/1614 ... done Processing lenders file 1108/1614 ... done Processing lenders file 1109/1614 ... done Processing lenders file 1110/1614 ... done Processing lenders file 1111/1614 ... done Processing lenders file 1112/1614 ... done Processing lenders file 1113/1614 ... done Processing lenders file 1114/1614 ... done Processing lenders file 1115/1614 ... done Processing lenders file 1116/1614 ... done Processing lenders file 1117/1614 ... done Processing lenders file 1118/1614 ... done Processing lenders file 1119/1614 ... done Processing lenders file 1120/1614 ... done Processing lenders file 1121/1614 ... done Processing lenders file 1122/1614 ... done Processing lenders file 1123/1614 ... done Processing lenders file 1124/1614 ... done Processing lenders file 1125/1614 ... done Processing lenders file 1126/1614 ... done Processing lenders file 1127/1614 ... done Processing lenders file 1128/1614 ... done Processing lenders file 1129/1614 ... done Processing lenders file 1130/1614 ... done Processing lenders file 1131/1614 ... done Processing lenders file 1132/1614 ... done Processing lenders file 1133/1614 ... done Processing lenders file 1134/1614 ... done Processing lenders file 1135/1614 ... done Processing lenders file 1136/1614 ... done Processing lenders file 1137/1614 ... done Processing lenders file 1138/1614 ... done Processing lenders file 1139/1614 ... done Processing lenders file 1140/1614 ... done Processing lenders file 1141/1614 ... done Processing lenders file 1142/1614 ... done Processing lenders file 1143/1614 ... done Processing lenders file 1144/1614 ... done Processing lenders file 1145/1614 ... done Processing lenders file 1146/1614 ... done Processing lenders file 1147/1614 ... done Processing lenders file 1148/1614 ... done Processing lenders file 1149/1614 ... done Processing lenders file 1150/1614 ... done Processing lenders file 1151/1614 ... done Processing lenders file 1152/1614 ... done Processing lenders file 1153/1614 ... done Processing lenders file 1154/1614 ... done Processing lenders file 1155/1614 ... done Processing lenders file 1156/1614 ... done Processing lenders file 1157/1614 ... done Processing lenders file 1158/1614 ... done Processing lenders file 1159/1614 ... done Processing lenders file 1160/1614 ... done Processing lenders file 1161/1614 ... done Processing lenders file 1162/1614 ... done Processing lenders file 1163/1614 ... done Processing lenders file 1164/1614 ... done Processing lenders file 1165/1614 ... done Processing lenders file 1166/1614 ... done Processing lenders file 1167/1614 ... done Processing lenders file 1168/1614 ... done Processing lenders file 1169/1614 ... done Processing lenders file 1170/1614 ... done Processing lenders file 1171/1614 ... done Processing lenders file 1172/1614 ... done Processing lenders file 1173/1614 ... done Processing lenders file 1174/1614 ... done Processing lenders file 1175/1614 ... done Processing lenders file 1176/1614 ... done Processing lenders file 1177/1614 ... done Processing lenders file 1178/1614 ... done Processing lenders file 1179/1614 ... done Processing lenders file 1180/1614 ... done Processing lenders file 1181/1614 ... done Processing lenders file 1182/1614 ... done Processing lenders file 1183/1614 ... done Processing lenders file 1184/1614 ... done Processing lenders file 1185/1614 ... done Processing lenders file 1186/1614 ... done Processing lenders file 1187/1614 ... done Processing lenders file 1188/1614 ... done Processing lenders file 1189/1614 ... done Processing lenders file 1190/1614 ... done Processing lenders file 1191/1614 ... done Processing lenders file 1192/1614 ... done Processing lenders file 1193/1614 ... done Processing lenders file 1194/1614 ... done Processing lenders file 1195/1614 ... done Processing lenders file 1196/1614 ... done Processing lenders file 1197/1614 ... done Processing lenders file 1198/1614 ... done Processing lenders file 1199/1614 ... done Processing lenders file 1200/1614 ... done Processing lenders file 1201/1614 ... done Processing lenders file 1202/1614 ... done Processing lenders file 1203/1614 ... done Processing lenders file 1204/1614 ... done Processing lenders file 1205/1614 ... done Processing lenders file 1206/1614 ... done Processing lenders file 1207/1614 ... done Processing lenders file 1208/1614 ... done Processing lenders file 1209/1614 ... done Processing lenders file 1210/1614 ... done Processing lenders file 1211/1614 ... done Processing lenders file 1212/1614 ... done Processing lenders file 1213/1614 ... done Processing lenders file 1214/1614 ... done Processing lenders file 1215/1614 ... done Processing lenders file 1216/1614 ... done Processing lenders file 1217/1614 ... done Processing lenders file 1218/1614 ... done Processing lenders file 1219/1614 ... done Processing lenders file 1220/1614 ... done Processing lenders file 1221/1614 ... done Processing lenders file 1222/1614 ... done Processing lenders file 1223/1614 ... done Processing lenders file 1224/1614 ... done Processing lenders file 1225/1614 ... done Processing lenders file 1226/1614 ... done Processing lenders file 1227/1614 ... done Processing lenders file 1228/1614 ... done Processing lenders file 1229/1614 ... done Processing lenders file 1230/1614 ... done Processing lenders file 1231/1614 ... done Processing lenders file 1232/1614 ... done Processing lenders file 1233/1614 ... done Processing lenders file 1234/1614 ... done Processing lenders file 1235/1614 ... done Processing lenders file 1236/1614 ... done Processing lenders file 1237/1614 ... done Processing lenders file 1238/1614 ... done Processing lenders file 1239/1614 ... done Processing lenders file 1240/1614 ... done Processing lenders file 1241/1614 ... done Processing lenders file 1242/1614 ... done Processing lenders file 1243/1614 ... done Processing lenders file 1244/1614 ... done Processing lenders file 1245/1614 ... done Processing lenders file 1246/1614 ... done Processing lenders file 1247/1614 ... done Processing lenders file 1248/1614 ... done Processing lenders file 1249/1614 ... done Processing lenders file 1250/1614 ... done Processing lenders file 1251/1614 ... done Processing lenders file 1252/1614 ... done Processing lenders file 1253/1614 ... done Processing lenders file 1254/1614 ... done Processing lenders file 1255/1614 ... done Processing lenders file 1256/1614 ... done Processing lenders file 1257/1614 ... done Processing lenders file 1258/1614 ... done Processing lenders file 1259/1614 ... done Processing lenders file 1260/1614 ... done Processing lenders file 1261/1614 ... done Processing lenders file 1262/1614 ... done Processing lenders file 1263/1614 ... done Processing lenders file 1264/1614 ... done Processing lenders file 1265/1614 ... done Processing lenders file 1266/1614 ... done Processing lenders file 1267/1614 ... done Processing lenders file 1268/1614 ... done Processing lenders file 1269/1614 ... done Processing lenders file 1270/1614 ... done Processing lenders file 1271/1614 ... done Processing lenders file 1272/1614 ... done Processing lenders file 1273/1614 ... done Processing lenders file 1274/1614 ... done Processing lenders file 1275/1614 ... done Processing lenders file 1276/1614 ... done Processing lenders file 1277/1614 ... done Processing lenders file 1278/1614 ... done Processing lenders file 1279/1614 ... done Processing lenders file 1280/1614 ... done Processing lenders file 1281/1614 ... done Processing lenders file 1282/1614 ... done Processing lenders file 1283/1614 ... done Processing lenders file 1284/1614 ... done Processing lenders file 1285/1614 ... done Processing lenders file 1286/1614 ... done Processing lenders file 1287/1614 ... done Processing lenders file 1288/1614 ... done Processing lenders file 1289/1614 ... done Processing lenders file 1290/1614 ... done Processing lenders file 1291/1614 ... done Processing lenders file 1292/1614 ... done Processing lenders file 1293/1614 ... done Processing lenders file 1294/1614 ... done Processing lenders file 1295/1614 ... done Processing lenders file 1296/1614 ... done Processing lenders file 1297/1614 ... done Processing lenders file 1298/1614 ... done Processing lenders file 1299/1614 ... done Processing lenders file 1300/1614 ... done Processing lenders file 1301/1614 ... done Processing lenders file 1302/1614 ... done Processing lenders file 1303/1614 ... done Processing lenders file 1304/1614 ... done Processing lenders file 1305/1614 ... done Processing lenders file 1306/1614 ... done Processing lenders file 1307/1614 ... done Processing lenders file 1308/1614 ... done Processing lenders file 1309/1614 ... done Processing lenders file 1310/1614 ... done Processing lenders file 1311/1614 ... done Processing lenders file 1312/1614 ... done Processing lenders file 1313/1614 ... done Processing lenders file 1314/1614 ... done Processing lenders file 1315/1614 ... done Processing lenders file 1316/1614 ... done Processing lenders file 1317/1614 ... done Processing lenders file 1318/1614 ... done Processing lenders file 1319/1614 ... done Processing lenders file 1320/1614 ... done Processing lenders file 1321/1614 ... done Processing lenders file 1322/1614 ... done Processing lenders file 1323/1614 ... done Processing lenders file 1324/1614 ... done Processing lenders file 1325/1614 ... done Processing lenders file 1326/1614 ... done Processing lenders file 1327/1614 ... done Processing lenders file 1328/1614 ... done Processing lenders file 1329/1614 ... done Processing lenders file 1330/1614 ... done Processing lenders file 1331/1614 ... done Processing lenders file 1332/1614 ... done Processing lenders file 1333/1614 ... done Processing lenders file 1334/1614 ... done Processing lenders file 1335/1614 ... done Processing lenders file 1336/1614 ... done Processing lenders file 1337/1614 ... done Processing lenders file 1338/1614 ... done Processing lenders file 1339/1614 ... done Processing lenders file 1340/1614 ... done Processing lenders file 1341/1614 ... done Processing lenders file 1342/1614 ... done Processing lenders file 1343/1614 ... done Processing lenders file 1344/1614 ... done Processing lenders file 1345/1614 ... done Processing lenders file 1346/1614 ... done Processing lenders file 1347/1614 ... done Processing lenders file 1348/1614 ... done Processing lenders file 1349/1614 ... done Processing lenders file 1350/1614 ... done Processing lenders file 1351/1614 ... done Processing lenders file 1352/1614 ... done Processing lenders file 1353/1614 ... done Processing lenders file 1354/1614 ... done Processing lenders file 1355/1614 ... done Processing lenders file 1356/1614 ... done Processing lenders file 1357/1614 ... done Processing lenders file 1358/1614 ... done Processing lenders file 1359/1614 ... done Processing lenders file 1360/1614 ... done Processing lenders file 1361/1614 ... done Processing lenders file 1362/1614 ... done Processing lenders file 1363/1614 ... done Processing lenders file 1364/1614 ... done Processing lenders file 1365/1614 ... done Processing lenders file 1366/1614 ... done Processing lenders file 1367/1614 ... done Processing lenders file 1368/1614 ... done Processing lenders file 1369/1614 ... done Processing lenders file 1370/1614 ... done Processing lenders file 1371/1614 ... done Processing lenders file 1372/1614 ... done Processing lenders file 1373/1614 ... done Processing lenders file 1374/1614 ... done Processing lenders file 1375/1614 ... done Processing lenders file 1376/1614 ... done Processing lenders file 1377/1614 ... done Processing lenders file 1378/1614 ... done Processing lenders file 1379/1614 ... done Processing lenders file 1380/1614 ... done Processing lenders file 1381/1614 ... done Processing lenders file 1382/1614 ... done Processing lenders file 1383/1614 ... done Processing lenders file 1384/1614 ... done Processing lenders file 1385/1614 ... done Processing lenders file 1386/1614 ... done Processing lenders file 1387/1614 ... done Processing lenders file 1388/1614 ... done Processing lenders file 1389/1614 ... done Processing lenders file 1390/1614 ... done Processing lenders file 1391/1614 ... done Processing lenders file 1392/1614 ... done Processing lenders file 1393/1614 ... done Processing lenders file 1394/1614 ... done Processing lenders file 1395/1614 ... done Processing lenders file 1396/1614 ... done Processing lenders file 1397/1614 ... done Processing lenders file 1398/1614 ... done Processing lenders file 1399/1614 ... done Processing lenders file 1400/1614 ... done Processing lenders file 1401/1614 ... done Processing lenders file 1402/1614 ... done Processing lenders file 1403/1614 ... done Processing lenders file 1404/1614 ... done Processing lenders file 1405/1614 ... done Processing lenders file 1406/1614 ... done Processing lenders file 1407/1614 ... done Processing lenders file 1408/1614 ... done Processing lenders file 1409/1614 ... done Processing lenders file 1410/1614 ... done Processing lenders file 1411/1614 ... done Processing lenders file 1412/1614 ... done Processing lenders file 1413/1614 ... done Processing lenders file 1414/1614 ... done Processing lenders file 1415/1614 ... done Processing lenders file 1416/1614 ... done Processing lenders file 1417/1614 ... done Processing lenders file 1418/1614 ... done Processing lenders file 1419/1614 ... done Processing lenders file 1420/1614 ... done Processing lenders file 1421/1614 ... done Processing lenders file 1422/1614 ... done Processing lenders file 1423/1614 ... done Processing lenders file 1424/1614 ... done Processing lenders file 1425/1614 ... done Processing lenders file 1426/1614 ... done Processing lenders file 1427/1614 ... done Processing lenders file 1428/1614 ... done Processing lenders file 1429/1614 ... done Processing lenders file 1430/1614 ... done Processing lenders file 1431/1614 ... done Processing lenders file 1432/1614 ... done Processing lenders file 1433/1614 ... done Processing lenders file 1434/1614 ... done Processing lenders file 1435/1614 ... done Processing lenders file 1436/1614 ... done Processing lenders file 1437/1614 ... done Processing lenders file 1438/1614 ... done Processing lenders file 1439/1614 ... done Processing lenders file 1440/1614 ... done Processing lenders file 1441/1614 ... done Processing lenders file 1442/1614 ... done Processing lenders file 1443/1614 ... done Processing lenders file 1444/1614 ... done Processing lenders file 1445/1614 ... done Processing lenders file 1446/1614 ... done Processing lenders file 1447/1614 ... done Processing lenders file 1448/1614 ... done Processing lenders file 1449/1614 ... done Processing lenders file 1450/1614 ... done Processing lenders file 1451/1614 ... done Processing lenders file 1452/1614 ... done Processing lenders file 1453/1614 ... done Processing lenders file 1454/1614 ... done Processing lenders file 1455/1614 ... done Processing lenders file 1456/1614 ... done Processing lenders file 1457/1614 ... done Processing lenders file 1458/1614 ... done Processing lenders file 1459/1614 ... done Processing lenders file 1460/1614 ... done Processing lenders file 1461/1614 ... done Processing lenders file 1462/1614 ... done Processing lenders file 1463/1614 ... done Processing lenders file 1464/1614 ... done Processing lenders file 1465/1614 ... done Processing lenders file 1466/1614 ... done Processing lenders file 1467/1614 ... done Processing lenders file 1468/1614 ... done Processing lenders file 1469/1614 ... done Processing lenders file 1470/1614 ... done Processing lenders file 1471/1614 ... done Processing lenders file 1472/1614 ... done Processing lenders file 1473/1614 ... done Processing lenders file 1474/1614 ... done Processing lenders file 1475/1614 ... done Processing lenders file 1476/1614 ... done Processing lenders file 1477/1614 ... done Processing lenders file 1478/1614 ... done Processing lenders file 1479/1614 ... done Processing lenders file 1480/1614 ... done Processing lenders file 1481/1614 ... done Processing lenders file 1482/1614 ... done Processing lenders file 1483/1614 ... done Processing lenders file 1484/1614 ... done Processing lenders file 1485/1614 ... done Processing lenders file 1486/1614 ... done Processing lenders file 1487/1614 ... done Processing lenders file 1488/1614 ... done Processing lenders file 1489/1614 ... done Processing lenders file 1490/1614 ... done Processing lenders file 1491/1614 ... done Processing lenders file 1492/1614 ... done Processing lenders file 1493/1614 ... done Processing lenders file 1494/1614 ... done Processing lenders file 1495/1614 ... done Processing lenders file 1496/1614 ... done Processing lenders file 1497/1614 ... done Processing lenders file 1498/1614 ... done Processing lenders file 1499/1614 ... done Processing lenders file 1500/1614 ... done Processing lenders file 1501/1614 ... done Processing lenders file 1502/1614 ... done Processing lenders file 1503/1614 ... done Processing lenders file 1504/1614 ... done Processing lenders file 1505/1614 ... done Processing lenders file 1506/1614 ... done Processing lenders file 1507/1614 ... done Processing lenders file 1508/1614 ... done Processing lenders file 1509/1614 ... done Processing lenders file 1510/1614 ... done Processing lenders file 1511/1614 ... done Processing lenders file 1512/1614 ... done Processing lenders file 1513/1614 ... done Processing lenders file 1514/1614 ... done Processing lenders file 1515/1614 ... done Processing lenders file 1516/1614 ... done Processing lenders file 1517/1614 ... done Processing lenders file 1518/1614 ... done Processing lenders file 1519/1614 ... done Processing lenders file 1520/1614 ... done Processing lenders file 1521/1614 ... done Processing lenders file 1522/1614 ... done Processing lenders file 1523/1614 ... done Processing lenders file 1524/1614 ... done Processing lenders file 1525/1614 ... done Processing lenders file 1526/1614 ... done Processing lenders file 1527/1614 ... done Processing lenders file 1528/1614 ... done Processing lenders file 1529/1614 ... done Processing lenders file 1530/1614 ... done Processing lenders file 1531/1614 ... done Processing lenders file 1532/1614 ... done Processing lenders file 1533/1614 ... done Processing lenders file 1534/1614 ... done Processing lenders file 1535/1614 ... done Processing lenders file 1536/1614 ... done Processing lenders file 1537/1614 ... done Processing lenders file 1538/1614 ... done Processing lenders file 1539/1614 ... done Processing lenders file 1540/1614 ... done Processing lenders file 1541/1614 ... done Processing lenders file 1542/1614 ... done Processing lenders file 1543/1614 ... done Processing lenders file 1544/1614 ... done Processing lenders file 1545/1614 ... done Processing lenders file 1546/1614 ... done Processing lenders file 1547/1614 ... done Processing lenders file 1548/1614 ... done Processing lenders file 1549/1614 ... done Processing lenders file 1550/1614 ... done Processing lenders file 1551/1614 ... done Processing lenders file 1552/1614 ... done Processing lenders file 1553/1614 ... done Processing lenders file 1554/1614 ... done Processing lenders file 1555/1614 ... done Processing lenders file 1556/1614 ... done Processing lenders file 1557/1614 ... done Processing lenders file 1558/1614 ... done Processing lenders file 1559/1614 ... done Processing lenders file 1560/1614 ... done Processing lenders file 1561/1614 ... done Processing lenders file 1562/1614 ... done Processing lenders file 1563/1614 ... done Processing lenders file 1564/1614 ... done Processing lenders file 1565/1614 ... done Processing lenders file 1566/1614 ... done Processing lenders file 1567/1614 ... done Processing lenders file 1568/1614 ... done Processing lenders file 1569/1614 ... done Processing lenders file 1570/1614 ... done Processing lenders file 1571/1614 ... done Processing lenders file 1572/1614 ... done Processing lenders file 1573/1614 ... done Processing lenders file 1574/1614 ... done Processing lenders file 1575/1614 ... done Processing lenders file 1576/1614 ... done Processing lenders file 1577/1614 ... done Processing lenders file 1578/1614 ... done Processing lenders file 1579/1614 ... done Processing lenders file 1580/1614 ... done Processing lenders file 1581/1614 ... done Processing lenders file 1582/1614 ... done Processing lenders file 1583/1614 ... done Processing lenders file 1584/1614 ... done Processing lenders file 1585/1614 ... done Processing lenders file 1586/1614 ... done Processing lenders file 1587/1614 ... done Processing lenders file 1588/1614 ... done Processing lenders file 1589/1614 ... done Processing lenders file 1590/1614 ... done Processing lenders file 1591/1614 ... done Processing lenders file 1592/1614 ... done Processing lenders file 1593/1614 ... done Processing lenders file 1594/1614 ... done Processing lenders file 1595/1614 ... done Processing lenders file 1596/1614 ... done Processing lenders file 1597/1614 ... done Processing lenders file 1598/1614 ... done Processing lenders file 1599/1614 ... done Processing lenders file 1600/1614 ... done Processing lenders file 1601/1614 ... done Processing lenders file 1602/1614 ... done Processing lenders file 1603/1614 ... done Processing lenders file 1604/1614 ... done Processing lenders file 1605/1614 ... done Processing lenders file 1606/1614 ... done Processing lenders file 1607/1614 ... done Processing lenders file 1608/1614 ... done Processing lenders file 1609/1614 ... done Processing lenders file 1610/1614 ... done Processing lenders file 1611/1614 ... done Processing lenders file 1612/1614 ... done Processing lenders file 1613/1614 ... done Processing lenders file 1614/1614 ... done
# Next line commented out because we only want to run this once
#!python src/load_kiva_loans_lenders_to_mongodb.py
Processing loans/lenders file 1/528 ... done Processing loans/lenders file 2/528 ... done Processing loans/lenders file 3/528 ... done Processing loans/lenders file 4/528 ... done Processing loans/lenders file 5/528 ... done Processing loans/lenders file 6/528 ... done Processing loans/lenders file 7/528 ... done Processing loans/lenders file 8/528 ... done Processing loans/lenders file 9/528 ... done Processing loans/lenders file 10/528 ... done Processing loans/lenders file 11/528 ... done Processing loans/lenders file 12/528 ... done Processing loans/lenders file 13/528 ... done Processing loans/lenders file 14/528 ... done Processing loans/lenders file 15/528 ... done Processing loans/lenders file 16/528 ... done Processing loans/lenders file 17/528 ... done Processing loans/lenders file 18/528 ... done Processing loans/lenders file 19/528 ... done Processing loans/lenders file 20/528 ... done Processing loans/lenders file 21/528 ... done Processing loans/lenders file 22/528 ... done Processing loans/lenders file 23/528 ... done Processing loans/lenders file 24/528 ... done Processing loans/lenders file 25/528 ... done Processing loans/lenders file 26/528 ... done Processing loans/lenders file 27/528 ... done Processing loans/lenders file 28/528 ... done Processing loans/lenders file 29/528 ... done Processing loans/lenders file 30/528 ... done Processing loans/lenders file 31/528 ... done Processing loans/lenders file 32/528 ... done Processing loans/lenders file 33/528 ... done Processing loans/lenders file 34/528 ... done Processing loans/lenders file 35/528 ... done Processing loans/lenders file 36/528 ... done Processing loans/lenders file 37/528 ... done Processing loans/lenders file 38/528 ... done Processing loans/lenders file 39/528 ... done Processing loans/lenders file 40/528 ... done Processing loans/lenders file 41/528 ... done Processing loans/lenders file 42/528 ... done Processing loans/lenders file 43/528 ... done Processing loans/lenders file 44/528 ... done Processing loans/lenders file 45/528 ... done Processing loans/lenders file 46/528 ... done Processing loans/lenders file 47/528 ... done Processing loans/lenders file 48/528 ... done Processing loans/lenders file 49/528 ... done Processing loans/lenders file 50/528 ... done Processing loans/lenders file 51/528 ... done Processing loans/lenders file 52/528 ... done Processing loans/lenders file 53/528 ... done Processing loans/lenders file 54/528 ... done Processing loans/lenders file 55/528 ... done Processing loans/lenders file 56/528 ... done Processing loans/lenders file 57/528 ... done Processing loans/lenders file 58/528 ... done Processing loans/lenders file 59/528 ... done Processing loans/lenders file 60/528 ... done Processing loans/lenders file 61/528 ... done Processing loans/lenders file 62/528 ... done Processing loans/lenders file 63/528 ... done Processing loans/lenders file 64/528 ... done Processing loans/lenders file 65/528 ... done Processing loans/lenders file 66/528 ... done Processing loans/lenders file 67/528 ... done Processing loans/lenders file 68/528 ... done Processing loans/lenders file 69/528 ... done Processing loans/lenders file 70/528 ... done Processing loans/lenders file 71/528 ... done Processing loans/lenders file 72/528 ... done Processing loans/lenders file 73/528 ... done Processing loans/lenders file 74/528 ... done Processing loans/lenders file 75/528 ... done Processing loans/lenders file 76/528 ... done Processing loans/lenders file 77/528 ... done Processing loans/lenders file 78/528 ... done Processing loans/lenders file 79/528 ... done Processing loans/lenders file 80/528 ... done Processing loans/lenders file 81/528 ... done Processing loans/lenders file 82/528 ... done Processing loans/lenders file 83/528 ... done Processing loans/lenders file 84/528 ... done Processing loans/lenders file 85/528 ... done Processing loans/lenders file 86/528 ... done Processing loans/lenders file 87/528 ... done Processing loans/lenders file 88/528 ... done Processing loans/lenders file 89/528 ... done Processing loans/lenders file 90/528 ... done Processing loans/lenders file 91/528 ... done Processing loans/lenders file 92/528 ... done Processing loans/lenders file 93/528 ... done Processing loans/lenders file 94/528 ... done Processing loans/lenders file 95/528 ... done Processing loans/lenders file 96/528 ... done Processing loans/lenders file 97/528 ... done Processing loans/lenders file 98/528 ... done Processing loans/lenders file 99/528 ... done Processing loans/lenders file 100/528 ... done Processing loans/lenders file 101/528 ... done Processing loans/lenders file 102/528 ... done Processing loans/lenders file 103/528 ... done Processing loans/lenders file 104/528 ... done Processing loans/lenders file 105/528 ... done Processing loans/lenders file 106/528 ... done Processing loans/lenders file 107/528 ... done Processing loans/lenders file 108/528 ... done Processing loans/lenders file 109/528 ... done Processing loans/lenders file 110/528 ... done Processing loans/lenders file 111/528 ... done Processing loans/lenders file 112/528 ... done Processing loans/lenders file 113/528 ... done Processing loans/lenders file 114/528 ... done Processing loans/lenders file 115/528 ... done Processing loans/lenders file 116/528 ... done Processing loans/lenders file 117/528 ... done Processing loans/lenders file 118/528 ... done Processing loans/lenders file 119/528 ... done Processing loans/lenders file 120/528 ... done Processing loans/lenders file 121/528 ... done Processing loans/lenders file 122/528 ... done Processing loans/lenders file 123/528 ... done Processing loans/lenders file 124/528 ... done Processing loans/lenders file 125/528 ... done Processing loans/lenders file 126/528 ... done Processing loans/lenders file 127/528 ... done Processing loans/lenders file 128/528 ... done Processing loans/lenders file 129/528 ... done Processing loans/lenders file 130/528 ... done Processing loans/lenders file 131/528 ... done Processing loans/lenders file 132/528 ... done Processing loans/lenders file 133/528 ... done Processing loans/lenders file 134/528 ... done Processing loans/lenders file 135/528 ... done Processing loans/lenders file 136/528 ... done Processing loans/lenders file 137/528 ... done Processing loans/lenders file 138/528 ... done Processing loans/lenders file 139/528 ... done Processing loans/lenders file 140/528 ... done Processing loans/lenders file 141/528 ... done Processing loans/lenders file 142/528 ... done Processing loans/lenders file 143/528 ... done Processing loans/lenders file 144/528 ... done Processing loans/lenders file 145/528 ... done Processing loans/lenders file 146/528 ... done Processing loans/lenders file 147/528 ... done Processing loans/lenders file 148/528 ... done Processing loans/lenders file 149/528 ... done Processing loans/lenders file 150/528 ... done Processing loans/lenders file 151/528 ... done Processing loans/lenders file 152/528 ... done Processing loans/lenders file 153/528 ... done Processing loans/lenders file 154/528 ... done Processing loans/lenders file 155/528 ... done Processing loans/lenders file 156/528 ... done Processing loans/lenders file 157/528 ... done Processing loans/lenders file 158/528 ... done Processing loans/lenders file 159/528 ... done Processing loans/lenders file 160/528 ... done Processing loans/lenders file 161/528 ... done Processing loans/lenders file 162/528 ... done Processing loans/lenders file 163/528 ... done Processing loans/lenders file 164/528 ... done Processing loans/lenders file 165/528 ... done Processing loans/lenders file 166/528 ... done Processing loans/lenders file 167/528 ... done Processing loans/lenders file 168/528 ... done Processing loans/lenders file 169/528 ... done Processing loans/lenders file 170/528 ... done Processing loans/lenders file 171/528 ... done Processing loans/lenders file 172/528 ... done Processing loans/lenders file 173/528 ... done Processing loans/lenders file 174/528 ... done Processing loans/lenders file 175/528 ... done Processing loans/lenders file 176/528 ... done Processing loans/lenders file 177/528 ... done Processing loans/lenders file 178/528 ... done Processing loans/lenders file 179/528 ... done Processing loans/lenders file 180/528 ... done Processing loans/lenders file 181/528 ... done Processing loans/lenders file 182/528 ... done Processing loans/lenders file 183/528 ... done Processing loans/lenders file 184/528 ... done Processing loans/lenders file 185/528 ... done Processing loans/lenders file 186/528 ... done Processing loans/lenders file 187/528 ... done Processing loans/lenders file 188/528 ... done Processing loans/lenders file 189/528 ... done Processing loans/lenders file 190/528 ... done Processing loans/lenders file 191/528 ... done Processing loans/lenders file 192/528 ... done Processing loans/lenders file 193/528 ... done Processing loans/lenders file 194/528 ... done Processing loans/lenders file 195/528 ... done Processing loans/lenders file 196/528 ... done Processing loans/lenders file 197/528 ... done Processing loans/lenders file 198/528 ... done Processing loans/lenders file 199/528 ... done Processing loans/lenders file 200/528 ... done Processing loans/lenders file 201/528 ... done Processing loans/lenders file 202/528 ... done Processing loans/lenders file 203/528 ... done Processing loans/lenders file 204/528 ... done Processing loans/lenders file 205/528 ... done Processing loans/lenders file 206/528 ... done Processing loans/lenders file 207/528 ... done Processing loans/lenders file 208/528 ... done Processing loans/lenders file 209/528 ... done Processing loans/lenders file 210/528 ... done Processing loans/lenders file 211/528 ... done Processing loans/lenders file 212/528 ... done Processing loans/lenders file 213/528 ... done Processing loans/lenders file 214/528 ... done Processing loans/lenders file 215/528 ... done Processing loans/lenders file 216/528 ... done Processing loans/lenders file 217/528 ... done Processing loans/lenders file 218/528 ... done Processing loans/lenders file 219/528 ... done Processing loans/lenders file 220/528 ... done Processing loans/lenders file 221/528 ... done Processing loans/lenders file 222/528 ... done Processing loans/lenders file 223/528 ... done Processing loans/lenders file 224/528 ... done Processing loans/lenders file 225/528 ... done Processing loans/lenders file 226/528 ... done Processing loans/lenders file 227/528 ... done Processing loans/lenders file 228/528 ... done Processing loans/lenders file 229/528 ... done Processing loans/lenders file 230/528 ... done Processing loans/lenders file 231/528 ... done Processing loans/lenders file 232/528 ... done Processing loans/lenders file 233/528 ... done Processing loans/lenders file 234/528 ... done Processing loans/lenders file 235/528 ... done Processing loans/lenders file 236/528 ... done Processing loans/lenders file 237/528 ... done Processing loans/lenders file 238/528 ... done Processing loans/lenders file 239/528 ... done Processing loans/lenders file 240/528 ... done Processing loans/lenders file 241/528 ... done Processing loans/lenders file 242/528 ... done Processing loans/lenders file 243/528 ... done Processing loans/lenders file 244/528 ... done Processing loans/lenders file 245/528 ... done Processing loans/lenders file 246/528 ... done Processing loans/lenders file 247/528 ... done Processing loans/lenders file 248/528 ... done Processing loans/lenders file 249/528 ... done Processing loans/lenders file 250/528 ... done Processing loans/lenders file 251/528 ... done Processing loans/lenders file 252/528 ... done Processing loans/lenders file 253/528 ... done Processing loans/lenders file 254/528 ... done Processing loans/lenders file 255/528 ... done Processing loans/lenders file 256/528 ... done Processing loans/lenders file 257/528 ... done Processing loans/lenders file 258/528 ... done Processing loans/lenders file 259/528 ... done Processing loans/lenders file 260/528 ... done Processing loans/lenders file 261/528 ... done Processing loans/lenders file 262/528 ... done Processing loans/lenders file 263/528 ... done Processing loans/lenders file 264/528 ... done Processing loans/lenders file 265/528 ... done Processing loans/lenders file 266/528 ... done Processing loans/lenders file 267/528 ... done Processing loans/lenders file 268/528 ... done Processing loans/lenders file 269/528 ... done Processing loans/lenders file 270/528 ... done Processing loans/lenders file 271/528 ... done Processing loans/lenders file 272/528 ... done Processing loans/lenders file 273/528 ... done Processing loans/lenders file 274/528 ... done Processing loans/lenders file 275/528 ... done Processing loans/lenders file 276/528 ... done Processing loans/lenders file 277/528 ... done Processing loans/lenders file 278/528 ... done Processing loans/lenders file 279/528 ... done Processing loans/lenders file 280/528 ... done Processing loans/lenders file 281/528 ... done Processing loans/lenders file 282/528 ... done Processing loans/lenders file 283/528 ... done Processing loans/lenders file 284/528 ... done Processing loans/lenders file 285/528 ... done Processing loans/lenders file 286/528 ... done Processing loans/lenders file 287/528 ... done Processing loans/lenders file 288/528 ... done Processing loans/lenders file 289/528 ... done Processing loans/lenders file 290/528 ... done Processing loans/lenders file 291/528 ... done Processing loans/lenders file 292/528 ... done Processing loans/lenders file 293/528 ... done Processing loans/lenders file 294/528 ... done Processing loans/lenders file 295/528 ... done Processing loans/lenders file 296/528 ... done Processing loans/lenders file 297/528 ... done Processing loans/lenders file 298/528 ... done Processing loans/lenders file 299/528 ... done Processing loans/lenders file 300/528 ... done Processing loans/lenders file 301/528 ... done Processing loans/lenders file 302/528 ... done Processing loans/lenders file 303/528 ... done Processing loans/lenders file 304/528 ... done Processing loans/lenders file 305/528 ... done Processing loans/lenders file 306/528 ... done Processing loans/lenders file 307/528 ... done Processing loans/lenders file 308/528 ... done Processing loans/lenders file 309/528 ... done Processing loans/lenders file 310/528 ... done Processing loans/lenders file 311/528 ... done Processing loans/lenders file 312/528 ... done Processing loans/lenders file 313/528 ... done Processing loans/lenders file 314/528 ... done Processing loans/lenders file 315/528 ... done Processing loans/lenders file 316/528 ... done Processing loans/lenders file 317/528 ... done Processing loans/lenders file 318/528 ... done Processing loans/lenders file 319/528 ... done Processing loans/lenders file 320/528 ... done Processing loans/lenders file 321/528 ... done Processing loans/lenders file 322/528 ... done Processing loans/lenders file 323/528 ... done Processing loans/lenders file 324/528 ... done Processing loans/lenders file 325/528 ... done Processing loans/lenders file 326/528 ... done Processing loans/lenders file 327/528 ... done Processing loans/lenders file 328/528 ... done Processing loans/lenders file 329/528 ... done Processing loans/lenders file 330/528 ... done Processing loans/lenders file 331/528 ... done Processing loans/lenders file 332/528 ... done Processing loans/lenders file 333/528 ... done Processing loans/lenders file 334/528 ... done Processing loans/lenders file 335/528 ... done Processing loans/lenders file 336/528 ... done Processing loans/lenders file 337/528 ... done Processing loans/lenders file 338/528 ... done Processing loans/lenders file 339/528 ... done Processing loans/lenders file 340/528 ... done Processing loans/lenders file 341/528 ... done Processing loans/lenders file 342/528 ... done Processing loans/lenders file 343/528 ... done Processing loans/lenders file 344/528 ... done Processing loans/lenders file 345/528 ... done Processing loans/lenders file 346/528 ... done Processing loans/lenders file 347/528 ... done Processing loans/lenders file 348/528 ... done Processing loans/lenders file 349/528 ... done Processing loans/lenders file 350/528 ... done Processing loans/lenders file 351/528 ... done Processing loans/lenders file 352/528 ... done Processing loans/lenders file 353/528 ... done Processing loans/lenders file 354/528 ... done Processing loans/lenders file 355/528 ... done Processing loans/lenders file 356/528 ... done Processing loans/lenders file 357/528 ... done Processing loans/lenders file 358/528 ... done Processing loans/lenders file 359/528 ... done Processing loans/lenders file 360/528 ... done Processing loans/lenders file 361/528 ... done Processing loans/lenders file 362/528 ... done Processing loans/lenders file 363/528 ... done Processing loans/lenders file 364/528 ... done Processing loans/lenders file 365/528 ... done Processing loans/lenders file 366/528 ... done Processing loans/lenders file 367/528 ... done Processing loans/lenders file 368/528 ... done Processing loans/lenders file 369/528 ... done Processing loans/lenders file 370/528 ... done Processing loans/lenders file 371/528 ... done Processing loans/lenders file 372/528 ... done Processing loans/lenders file 373/528 ... done Processing loans/lenders file 374/528 ... done Processing loans/lenders file 375/528 ... done Processing loans/lenders file 376/528 ... done Processing loans/lenders file 377/528 ... done Processing loans/lenders file 378/528 ... done Processing loans/lenders file 379/528 ... done Processing loans/lenders file 380/528 ... done Processing loans/lenders file 381/528 ... done Processing loans/lenders file 382/528 ... done Processing loans/lenders file 383/528 ... done Processing loans/lenders file 384/528 ... done Processing loans/lenders file 385/528 ... done Processing loans/lenders file 386/528 ... done Processing loans/lenders file 387/528 ... done Processing loans/lenders file 388/528 ... done Processing loans/lenders file 389/528 ... done Processing loans/lenders file 390/528 ... done Processing loans/lenders file 391/528 ... done Processing loans/lenders file 392/528 ... done Processing loans/lenders file 393/528 ... done Processing loans/lenders file 394/528 ... done Processing loans/lenders file 395/528 ... done Processing loans/lenders file 396/528 ... done Processing loans/lenders file 397/528 ... done Processing loans/lenders file 398/528 ... done Processing loans/lenders file 399/528 ... done Processing loans/lenders file 400/528 ... done Processing loans/lenders file 401/528 ... done Processing loans/lenders file 402/528 ... done Processing loans/lenders file 403/528 ... done Processing loans/lenders file 404/528 ... done Processing loans/lenders file 405/528 ... done Processing loans/lenders file 406/528 ... done Processing loans/lenders file 407/528 ... done Processing loans/lenders file 408/528 ... done Processing loans/lenders file 409/528 ... done Processing loans/lenders file 410/528 ... done Processing loans/lenders file 411/528 ... done Processing loans/lenders file 412/528 ... done Processing loans/lenders file 413/528 ... done Processing loans/lenders file 414/528 ... done Processing loans/lenders file 415/528 ... done Processing loans/lenders file 416/528 ... done Processing loans/lenders file 417/528 ... done Processing loans/lenders file 418/528 ... done Processing loans/lenders file 419/528 ... done Processing loans/lenders file 420/528 ... done Processing loans/lenders file 421/528 ... done Processing loans/lenders file 422/528 ... done Processing loans/lenders file 423/528 ... done Processing loans/lenders file 424/528 ... done Processing loans/lenders file 425/528 ... done Processing loans/lenders file 426/528 ... done Processing loans/lenders file 427/528 ... done Processing loans/lenders file 428/528 ... done Processing loans/lenders file 429/528 ... done Processing loans/lenders file 430/528 ... done Processing loans/lenders file 431/528 ... done Processing loans/lenders file 432/528 ... done Processing loans/lenders file 433/528 ... done Processing loans/lenders file 434/528 ... done Processing loans/lenders file 435/528 ... done Processing loans/lenders file 436/528 ... done Processing loans/lenders file 437/528 ... done Processing loans/lenders file 438/528 ... done Processing loans/lenders file 439/528 ... done Processing loans/lenders file 440/528 ... done Processing loans/lenders file 441/528 ... done Processing loans/lenders file 442/528 ... done Processing loans/lenders file 443/528 ... done Processing loans/lenders file 444/528 ... done Processing loans/lenders file 445/528 ... done Processing loans/lenders file 446/528 ... done Processing loans/lenders file 447/528 ... done Processing loans/lenders file 448/528 ... done Processing loans/lenders file 449/528 ... done Processing loans/lenders file 450/528 ... done Processing loans/lenders file 451/528 ... done Processing loans/lenders file 452/528 ... done Processing loans/lenders file 453/528 ... done Processing loans/lenders file 454/528 ... done Processing loans/lenders file 455/528 ... done Processing loans/lenders file 456/528 ... done Processing loans/lenders file 457/528 ... done Processing loans/lenders file 458/528 ... done Processing loans/lenders file 459/528 ... done Processing loans/lenders file 460/528 ... done Processing loans/lenders file 461/528 ... done Processing loans/lenders file 462/528 ... done Processing loans/lenders file 463/528 ... done Processing loans/lenders file 464/528 ... done Processing loans/lenders file 465/528 ... done Processing loans/lenders file 466/528 ... done Processing loans/lenders file 467/528 ... done Processing loans/lenders file 468/528 ... done Processing loans/lenders file 469/528 ... done Processing loans/lenders file 470/528 ... done Processing loans/lenders file 471/528 ... done Processing loans/lenders file 472/528 ... done Processing loans/lenders file 473/528 ... done Processing loans/lenders file 474/528 ... done Processing loans/lenders file 475/528 ... done Processing loans/lenders file 476/528 ... done Processing loans/lenders file 477/528 ... done Processing loans/lenders file 478/528 ... done Processing loans/lenders file 479/528 ... done Processing loans/lenders file 480/528 ... done Processing loans/lenders file 481/528 ... done Processing loans/lenders file 482/528 ... done Processing loans/lenders file 483/528 ... done Processing loans/lenders file 484/528 ... done Processing loans/lenders file 485/528 ... done Processing loans/lenders file 486/528 ... done Processing loans/lenders file 487/528 ... done Processing loans/lenders file 488/528 ... done Processing loans/lenders file 489/528 ... done Processing loans/lenders file 490/528 ... done Processing loans/lenders file 491/528 ... done Processing loans/lenders file 492/528 ... done Processing loans/lenders file 493/528 ... done Processing loans/lenders file 494/528 ... done Processing loans/lenders file 495/528 ... done Processing loans/lenders file 496/528 ... done Processing loans/lenders file 497/528 ... done Processing loans/lenders file 498/528 ... done Processing loans/lenders file 499/528 ... done Processing loans/lenders file 500/528 ... done Processing loans/lenders file 501/528 ... done Processing loans/lenders file 502/528 ... done Processing loans/lenders file 503/528 ... done Processing loans/lenders file 504/528 ... done Processing loans/lenders file 505/528 ... done Processing loans/lenders file 506/528 ... done Processing loans/lenders file 507/528 ... done Processing loans/lenders file 508/528 ... done Processing loans/lenders file 509/528 ... done Processing loans/lenders file 510/528 ... done Processing loans/lenders file 511/528 ... done Processing loans/lenders file 512/528 ... done Processing loans/lenders file 513/528 ... done Processing loans/lenders file 514/528 ... done Processing loans/lenders file 515/528 ... done Processing loans/lenders file 516/528 ... done Processing loans/lenders file 517/528 ... done Processing loans/lenders file 518/528 ... done Processing loans/lenders file 519/528 ... done Processing loans/lenders file 520/528 ... done Processing loans/lenders file 521/528 ... done Processing loans/lenders file 522/528 ... done Processing loans/lenders file 523/528 ... done Processing loans/lenders file 524/528 ... done Processing loans/lenders file 525/528 ... done Processing loans/lenders file 526/528 ... done Processing loans/lenders file 527/528 ... done Processing loans/lenders file 528/528 ... done
Before diving into predictive models, let's get to know our data a little better.
!python src/convert_mongodb_loans_to_dataframe.py --dataDir data/predicting_funding \
--baseName loans \
--startYear 2006 \
--endYear 2014
Creating MongoDB cursor ... done read 50000 documents ... read 100000 documents ... read 150000 documents ... read 200000 documents ... read 250000 documents ... read 300000 documents ... read 350000 documents ... read 400000 documents ... read 450000 documents ... read 500000 documents ... read 550000 documents ... read 600000 documents ... read 650000 documents ... read 700000 documents ... read 750000 documents ... Setting and sorting the dataframe index ... done Shape of dataframe: (756968, 7) Writing loans dataframe file data/predicting_funding/loans_dataframe.h5 .../usr/local/lib/python2.7/site-packages/pandas/io/pytables.py:2486: PerformanceWarning: your performance may suffer as PyTables will pickle object types that it cannot map directly to c-types [inferred_type->mixed,key->block2_values] [items->['borrower_majority_gender', 'planned_expiration_date', 'location_country_code']] warnings.warn(ws, PerformanceWarning) done
%matplotlib inline
import pandas as pd
import numpy as np
import sys
import matplotlib.pyplot as plt
from matplotlib.dates import date2num
from pandas.io.pytables import read_hdf
import matplotlib.dates as mdates
loansDataFrameFile = 'data/predicting_funding/loans_dataframe.h5'
print >> sys.stderr, "Reading loans dataframe file %s ..." % loansDataFrameFile,
loansDF = read_hdf(loansDataFrameFile, 'table')
print >> sys.stderr, "done"
#print "loansDF.shape = ", loansDF.shape
# Map date to the first day of the quarter
loansDF['posted_quarter'] = loansDF['posted_date'].apply( lambda x: x.replace(hour=0, minute=0, second=0, day=1, month=((x.month / 4)*3)+1 ))
loansDF['fully_funded'] = loansDF['funded_amount'] >= loansDF['loan_amount']
#loansDF.head(3)
loansDFGroupedByMonthAndGender = loansDF.groupby(['posted_quarter','borrower_majority_gender','fully_funded'])
XSeries = []
YSeries = {'M': {True: {},
False: {}},
'F': {True: {},
False: {}},
'N': {True: {},
False: {}}}
for k,v in sorted(loansDFGroupedByMonthAndGender.groups.items(), key=lambda x : x[0][0]):
period, gender, isFullyFunded = k
count = len(v)
XSeries.append(period)
YSeries[gender][isFullyFunded][period] = count
for p in XSeries:
for gender in ['M', 'F', 'N']:
for isFullyFunded in [True, False]:
if not YSeries[gender][isFullyFunded].has_key(p):
YSeries[gender][isFullyFunded][p] = 0
YSeriesMaleNotFullyFunded = [YSeries['M'][False][p] for p in XSeries]
YSeriesMaleFullyFunded = [YSeries['M'][True][p] for p in XSeries]
YSeriesFemaleNotFullyFunded = [YSeries['F'][False][p] for p in XSeries]
YSeriesFemaleFullyFunded = [YSeries['F'][True][p] for p in XSeries]
# Negligeable
#YSeriesNeutralNotFullyFunded = [YSeries['N'][False][p] for p in XSeries]
#YSeriesNeutralFullyFunded = [YSeries['N'][True][p] for p in XSeries]
width = 20
fig = plt.figure(40, figsize=(12,12))
ax = fig.add_subplot(111)
ax.bar(date2num(XSeries), YSeriesMaleNotFullyFunded,label="male, not fully funded",width=width,color='lightblue',align='center')
ax.bar(date2num(XSeries), YSeriesMaleFullyFunded,bottom=YSeriesMaleNotFullyFunded,label="male, fully funded",width=width,color='blue',align='center')
ax.bar(date2num(XSeries)+30, YSeriesFemaleNotFullyFunded,label="female, not fully funded", width=width,color='lightgreen',align='center')
ax.bar(date2num(XSeries)+30, YSeriesFemaleFullyFunded, bottom=YSeriesFemaleNotFullyFunded ,label="female, fully funded", width=width,color='green',align='center')
ax.xaxis_date()
plt.setp(plt.gca().get_yticklabels(), fontsize="25")
plt.setp(plt.gca().get_xticklabels(), fontsize="25", ha="left")
ax.autoscale(tight=True)
plt.legend(loc="best", fontsize="25")
plt.title('Number of loans per quarter\nsplit by gender and funding status', fontsize="25", y=1.03)
plt.ylabel('Number of loans', fontsize="25")
ax.yaxis.labelpad = 20
plt.show()
Reading loans dataframe file data/predicting_funding/loans_dataframe.h5 ... done
Learnings:
#print loansDF.head()
amountXSeries = []
amountYSeries = {'M': {True: {},
False: {}},
'F': {True: {},
False: {}},
'N': {True: {},
False: {}}}
for k,v in sorted(loansDFGroupedByMonthAndGender.groups.items(), key=lambda x : x[0][0]):
period, gender, isFullyFunded = k
row = loansDF.loc[v]
loanAmountSum = sum(row['loan_amount'])
# print "loanAmountSum = ", loanAmountSum
amountXSeries.append(period)
if amountYSeries[gender][isFullyFunded].has_key(period):
# print "increasing %s,%s,%s from %d to" % (gender, isFullyFunded, period, amountYSeries[gender][isFullyFunded][period]),
amountYSeries[gender][isFullyFunded][period] += loanAmountSum
# print "%d" % amountYSeries[gender][isFullyFunded][period]
else:
amountYSeries[gender][isFullyFunded][period] = loanAmountSum
# print "setting %s,%s,%s to %d" % (gender, isFullyFunded, period, amountYSeries[gender][isFullyFunded][period])
for p in amountXSeries:
for gender in ['M', 'F', 'N']:
for isFullyFunded in [True, False]:
if not amountYSeries[gender][isFullyFunded].has_key(p):
amountYSeries[gender][isFullyFunded][p] = 0
#print amountYSeries
amountYSeriesMaleNotFullyFunded = [amountYSeries['M'][False][p] for p in amountXSeries]
amountYSeriesMaleFullyFunded = [amountYSeries['M'][True][p] for p in amountXSeries]
amountYSeriesFemaleNotFullyFunded = [amountYSeries['F'][False][p] for p in amountXSeries]
amountYSeriesFemaleFullyFunded = [amountYSeries['F'][True][p] for p in amountXSeries]
# Negligeable
#amountYSeriesNeutralNotFullyFunded = [amountYSeries['N'][False][p] for p in amountXSeries]
#amountYSeriesNeutralFullyFunded = [amountYSeries['N'][True][p] for p in amountXSeries]
width = 20
fig = plt.figure(40, figsize=(12,12))
ax1 = fig.add_subplot(111)
ax1.bar(date2num(amountXSeries), amountYSeriesMaleNotFullyFunded,label="male, not fully funded",width=width,color='lightblue',align='center')
ax1.bar(date2num(amountXSeries), amountYSeriesMaleFullyFunded,bottom=amountYSeriesMaleNotFullyFunded,label="male, fully funded",width=width,color='blue',align='center')
ax1.bar(date2num(amountXSeries)+30, amountYSeriesFemaleNotFullyFunded,label="female, not fully funded", width=width,color='lightgreen',align='center')
ax1.bar(date2num(amountXSeries)+30, amountYSeriesFemaleFullyFunded, bottom=amountYSeriesFemaleNotFullyFunded ,label="female, fully funded", width=width,color='green',align='center')
ax1.xaxis_date()
plt.setp(plt.gca().get_yticklabels(), fontsize="20")
plt.setp(plt.gca().get_xticklabels(), fontsize="20", ha="left")
ax.autoscale(tight=True)
plt.legend(loc="best", fontsize="20")
plt.title('Total requested loan amounts per quarter\nsplit by gender and funding status', fontsize="20", y=1.03)
plt.ylabel('USD', fontsize="20")
ax.yaxis.labelpad = 20
plt.show()
The most notable difference between this graph (loan amounts) and the previous one (number of loans) is a downfall in 2012-2013. This suggests that the average loan amount in these years must have dropped - which is to be confirmed separately.
loanXSeries = {'M': {True: [],
False: []},
'F': {True: [],
False: []},
'N': {True: [],
False: []}}
loanYSeries = {'M': {True: {},
False: {}},
'F': {True: {},
False: {}},
'N': {True: {},
False: {}}}
for k,v in sorted(loansDFGroupedByMonthAndGender.groups.items(), key=lambda x : x[0][0]):
period, gender, isFullyFunded = k
row = loansDF.loc[v]
loanAmountAverage = sum(row['loan_amount']) / float(len(row['loan_amount']))
fundedAmountAverage = sum(row['funded_amount']) / float(len(row['funded_amount']))
lenderCountAverage = sum(row['lender_count']) / float(len(row['lender_count']))
loanXSeries[gender][isFullyFunded].append(period)
loanYSeries[gender][isFullyFunded][period] = {'loanAmountAvg': loanAmountAverage,
'fundedAmountAvg': fundedAmountAverage,
'lenderCountAvg' : lenderCountAverage}
loanYSeriesMaleNotFullyFunded = [loanYSeries['M'][False][p]['loanAmountAvg'] for p in loanXSeries['M'][False]]
loanYSeriesMaleFullyFunded = [loanYSeries['M'][True][p]['loanAmountAvg'] for p in loanXSeries['M'][True]]
fundedYSeriesMaleNotFullyFunded = [loanYSeries['M'][False][p]['fundedAmountAvg'] for p in loanXSeries['M'][False]]
lenderCountYSeriesMaleNotFullyFunded = [loanYSeries['M'][False][p]['lenderCountAvg'] for p in loanXSeries['M'][False]]
lenderCountYSeriesMaleFullyFunded = [loanYSeries['M'][True][p]['lenderCountAvg'] for p in loanXSeries['M'][True]]
loanYSeriesFemaleNotFullyFunded = [loanYSeries['F'][False][p]['loanAmountAvg'] for p in loanXSeries['F'][False]]
loanYSeriesFemaleFullyFunded = [loanYSeries['F'][True][p]['loanAmountAvg'] for p in loanXSeries['F'][True]]
fundedYSeriesFemaleNotFullyFunded = [loanYSeries['F'][False][p]['fundedAmountAvg'] for p in loanXSeries['F'][False]]
lenderCountYSeriesFemaleNotFullyFunded = [loanYSeries['F'][False][p]['lenderCountAvg'] for p in loanXSeries['F'][False]]
lenderCountYSeriesFemaleFullyFunded = [loanYSeries['F'][True][p]['lenderCountAvg'] for p in loanXSeries['F'][True]]
width = 20
fig = plt.figure(60, figsize=(12,8))
ax2 = fig.add_subplot(2,1,1)
ax2.plot(date2num(loanXSeries['M'][False]), loanYSeriesMaleNotFullyFunded, '+-', label="avg. loan amount, male, not fully funded", color='blue')
ax2.plot(date2num(loanXSeries['M'][False]), fundedYSeriesMaleNotFullyFunded, '*-', label="avg. funded amount, male, not fully funded", color='blue')
ax2.plot(date2num(loanXSeries['M'][True]), loanYSeriesMaleFullyFunded, 'o-', label="avg. loan=funded amount, male, fully funded", color='blue')
ax2.plot(date2num(loanXSeries['F'][False]), loanYSeriesFemaleNotFullyFunded, '+-', label="avg. loan amount, female, not fully funded", color='green')
ax2.plot(date2num(loanXSeries['F'][False]), fundedYSeriesFemaleNotFullyFunded, '*-', label="avg. funded amount, female, not fully funded", color='green')
ax2.plot(date2num(loanXSeries['F'][True]), loanYSeriesFemaleFullyFunded, 'o-', label="avg. loan=funded amount, female, fully funded", color='green')
ax2.xaxis_date()
ax2.autoscale(tight=True)
ax2.set_ylabel('USD')
ax2.legend(loc="best")
ax2.set_title('Loan characteristics per quarter, split by gender and funding status')
ax3 = fig.add_subplot(2,1,2)
ax3.plot(date2num(loanXSeries['M'][True]), lenderCountYSeriesMaleFullyFunded, 'o-', label="male, fully funded", color='blue')
ax3.plot(date2num(loanXSeries['F'][True]), lenderCountYSeriesFemaleFullyFunded, 'o-', label="female, fully funded", color='green')
ax3.plot(date2num(loanXSeries['M'][False]), lenderCountYSeriesMaleNotFullyFunded, 'x-', label="male, not fully funded", color='blue')
ax3.plot(date2num(loanXSeries['F'][False]), lenderCountYSeriesFemaleNotFullyFunded, 'x-', label="female, not fully funded", color='green')
ax3.set_ylabel("Number of lenders per loan")
ax3.xaxis_date()
ax3.autoscale(tight=True)
ax3.set_title('Average number of lenders per loan, split by gender and funding status')
ax3.legend(loc="best")
plt.show()
Learnings from the first graph:
And from the second:
!python src/calculate_country2country_loan_flow.py --inDataDir data/predicting_funding \
--isoCountryCodesFile data/predicting_funding/iso3166_country_codes.tsv \
--outDataDir data/predicting_funding \
--outBaseName kiva \
--startYear 2012 \
--endYear 2014 \
--minValue 5000
Loading ISO country codes from data/predicting_funding/iso3166_country_codes.tsv ... done Creating MongoDB cursor on lenders collection ... done >>>>>>>>>>>>>>>>>>>>>>> loaded 100000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 200000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 300000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 400000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 500000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 600000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 700000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 800000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 900000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 1000000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 1100000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 1200000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 1300000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 1400000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 1500000 lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 1600000 lenders ... Creating MongoDB cursor on loans_lenders collection ... done >>>>>>>>>>>>>>>>>>>>>>> loaded 100000 loan_lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 200000 loan_lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 300000 loan_lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 400000 loan_lenders ... >>>>>>>>>>>>>>>>>>>>>>> loaded 500000 loan_lenders ... Number of loans loaded from loans_lenders: 526559 Creating MongoDB cursor on loans collection ... done >>>>>>>>>>>>>>>>>>>>>>> processed 10000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 20000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 30000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 40000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 50000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 60000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 70000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 80000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 90000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 100000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 110000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 120000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 130000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 140000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 150000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 160000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 170000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 180000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 190000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 200000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 210000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 220000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 230000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 240000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 250000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 260000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 270000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 280000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 290000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 300000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 310000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 320000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 330000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 340000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 350000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 360000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 370000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 380000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 390000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 400000 fully funded loans ... >>>>>>>>>>>>>>>>>>>>>>> processed 410000 fully funded loans ... smallestLoanId = 353812 largestLoanId = 823023 nrLoanIdHits = 419769 nrLoanIdMisses = 144 nrLenderIdsHits = 418610 nrLenderIdsMisses = 1159 nrLenderIdHits = 9409344 nrLenderIdMisses = 383308 nrCountryHits = 7849493 nrCountryMisses = 1559851 8 lendingRegionCodes: ['oth', 'na', 'unk', 'eu', 'oc', 'as', 'sa', 'af'] 7 borrowingRegionCodes: ['oth', 'as', 'af', 'sa', 'na', 'eu', 'oc'] 74 lendingCountries: ['oth', u'bs', u'ca', u'cr', u'gt', u'mx', u'ni', u'pr', u'sv', u'us', 'unk', u'at', u'be', u'ch', u'cz', u'de', u'dk', u'es', u'fi', u'fr', u'gb', u'gr', u'hr', u'hu', u'ie', u'is', u'it', u'lu', u'mc', u'nl', u'no', u'pl', u'pt', u'ro', u'se', u'sk', u'tr', u'au', u'nz', u'ae', u'am', u'cn', u'hk', u'id', u'il', u'in', u'ir', u'jp', u'kg', u'kh', u'kr', u'kw', u'my', u'om', u'ph', u'pk', u'qa', u'sa', u'sg', u'th', u'tw', u'vn', u'vu', u'ye', u'ar', u'br', u'cl', u'co', u'ec', u'pe', u'py', u'ke', u'tz', u'za'] 78 borrowingCountries: ['oth', u'am', u'az', u'cn', u'ge', u'id', u'il', u'in', u'iq', u'jo', u'kg', u'kh', u'la', u'lb', u'mm', u'mn', u'np', u'ph', u'pk', u'ps', u'th', u'tj', u'tl', u'vn', u'ye', u'bf', u'bi', u'bj', u'cd', u'cg', u'ci', u'cm', u'eg', u'gh', u'ke', u'lr', u'ml', u'mr', u'mw', u'mz', u'ng', u'qs', u'rw', u'sl', u'sn', u'so', u'tg', u'tz', u'ug', u'za', u'zm', u'zw', u'bo', u'br', u'cl', u'co', u'ec', u'pe', u'py', u'sr', u'bz', u'cr', u'do', u'gt', u'hn', u'ht', u'mx', u'ni', u'pa', u'sv', u'us', u'al', u'md', u'tr', u'ua', u'xk', u'pg', u'ws'] Saving result in data/predicting_funding/kiva_country2country_loan_flows.json ... done
!cp data/predicting_funding/kiva_country2country_loan_flows.json d3/data.json
%%HTML
<iframe src="d3/sankey.html" width=1000 height=1650></iframe>
Before we build a data set for our classifier, we want to know to what extent the non-funded loans are complete, partial or near misses.
%matplotlib inline
import pandas as pd
import numpy as np
import sys
import matplotlib.pyplot as plt
from matplotlib.dates import date2num
from pandas.io.pytables import read_hdf
loansDataFrameFile = 'data/predicting_funding/loans_dataframe.h5'
print >> sys.stderr, "Reading loans dataframe file %s ..." % loansDataFrameFile,
loansDF = read_hdf(loansDataFrameFile, 'table')
print >> sys.stderr, "done"
# We're only interested in loans from the period 2012-2014 that were not fully funded
loansDF['funding_ratio'] = loansDF['funded_amount'] / loansDF['loan_amount']
nonFundedLoansDF = loansDF[loansDF['funding_ratio'] < 1.0]
#nonFundedLoansDF.head(3)
fundingRatioSeries = nonFundedLoansDF['funding_ratio'].values
print "number of not-fully funded loans in 2012-2014 period:", len(fundingRatioSeries)
plt.title("Distribution of funding ratio\nfor not-fully funded loans applications")
for nrBins in [2,3,4,5,10,20,100]:
plt.hist(fundingRatioSeries, nrBins, label="%d bins" % nrBins)
plt.legend()
plt.show()
Reading loans dataframe file data/predicting_funding/loans_dataframe.h5 ...
number of not-fully funded loans in 2012-2014 period: 26413
done
Even though at small granularity (100 bins) there is a peak of zero-funded loans applications, more than half of the not fully funded loans did manage to collect more than 50% of the requested loan sum. Apart from the 0-10% mode, there is another one in the region of 0.6. If we include the mode of all fully funded loan applications (not shown here), our data set may actually be fundamentally trimodal. We could express naturally express this as three possible answers to the funding question: "yes", "no" and "maybe".
Let's control this trimodality for gender:
nonFundedLoansFemaleDF = loansDF[(loansDF['funding_ratio'] < 1.0) & (loansDF['borrower_majority_gender'] == 'F')]
nonFundedLoansMaleDF = loansDF[(loansDF['funding_ratio'] < 1.0) & (loansDF['borrower_majority_gender'] == 'M')]
FemaleFundingRatioSeries = nonFundedLoansFemaleDF['funding_ratio'].values
MaleFundingRatioSeries = nonFundedLoansMaleDF['funding_ratio'].values
print "number of not-fully funded female loans in 2012-2014 period:", len(FemaleFundingRatioSeries)
print "number of not-fully funded male loans in 2012-2014 period:", len(MaleFundingRatioSeries)
plt.title("Distribution of funding ratio\nfor not-fully funded loans applications by FEMALES")
for nrBins in [2,3,4,5,10,20,100]:
plt.hist(FemaleFundingRatioSeries, nrBins, label="%d bins" % nrBins)
plt.legend()
plt.show()
plt.title("Distribution of funding ratio\nfor not-fully funded loans applications by MALES")
for nrBins in [2,3,4,5,10,20,100]:
plt.hist(MaleFundingRatioSeries, nrBins, label="%d bins" % nrBins)
plt.legend()
plt.show()
number of not-fully funded female loans in 2012-2014 period: 11964 number of not-fully funded male loans in 2012-2014 period: 14369
The trimodal structure holds for both female and male loan applications, but the existence of the zero mode (= flat refusal) is more outspoken for female than for male borrowers.
Key assumptions when building the data set:
There are two classification labels, directly calculated from the funding ratio F:
Label | Range | Fund loan? |
---|---|---|
0 | 0.00 <= F < 1.0 | No |
1 | 1.00 <= F | Yes |
!python src/convert_mongodb_to_blei_ldac.py --dataDir data/predicting_funding\
--corpusBaseName kiva \
--stopwordFile=data/predicting_funding/kiva_stopwords.tsv \
--startYear 2012 \
--endYear 2014 \
--maxNrDocs 1000000 \
--filterBelow 10 \
--filterAbove 0.5 \
--filterKeepN 1000 \
--classificationField funding_ratio \
--classLabelFileName kiva-labels-with-loan-ids.dat
Creating MongoDB cursor ... done Number of loans in 'en' between 2012 and 2014: 409530 First pass: streaming from MongoDB ... creating the dictionary ... read 5000 documents ... read 10000 documents ... read 15000 documents ... read 20000 documents ... read 25000 documents ... read 30000 documents ... read 35000 documents ... read 40000 documents ... read 45000 documents ... read 50000 documents ... read 55000 documents ... read 60000 documents ... read 65000 documents ... read 70000 documents ... read 75000 documents ... read 80000 documents ... read 85000 documents ... read 90000 documents ... read 95000 documents ... read 100000 documents ... read 105000 documents ... read 110000 documents ... read 115000 documents ... read 120000 documents ... read 125000 documents ... read 130000 documents ... read 135000 documents ... read 140000 documents ... read 145000 documents ... read 150000 documents ... read 155000 documents ... read 160000 documents ... read 165000 documents ... read 170000 documents ... read 175000 documents ... read 180000 documents ... read 185000 documents ... read 190000 documents ... read 195000 documents ... read 200000 documents ... read 205000 documents ... read 210000 documents ... read 215000 documents ... read 220000 documents ... read 225000 documents ... read 230000 documents ... read 235000 documents ... read 240000 documents ... read 245000 documents ... read 250000 documents ... read 255000 documents ... read 260000 documents ... read 265000 documents ... read 270000 documents ... read 275000 documents ... read 280000 documents ... read 285000 documents ... read 290000 documents ... read 295000 documents ... read 300000 documents ... read 305000 documents ... read 310000 documents ... read 315000 documents ... read 320000 documents ... read 325000 documents ... read 330000 documents ... read 335000 documents ... read 340000 documents ... read 345000 documents ... read 350000 documents ... read 355000 documents ... read 360000 documents ... read 365000 documents ... read 370000 documents ... read 375000 documents ... read 380000 documents ... read 385000 documents ... read 390000 documents ... read 395000 documents ... read 400000 documents ... read 405000 documents ... filtering the dictionary ... done wrote data/predicting_funding/kiva_dict.bin ... and data/predicting_funding/kiva_dict.txt ... done Second pass: streaming from MongoDB ... saving into data/predicting_funding/kiva.lda-c (Blei corpus format) ... done Number of documents converted: 409530 Vocabulary size: 1000
!python src/prepare_sdla_train_test_sets.py --bleiCorpusFile data/predicting_funding/kiva.lda-c \
--bleiLabelFile data/predicting_funding/kiva-labels-with-loan-ids.dat \
--bleiTrainCorpusFile data/predicting_funding/slda_in/kiva-train-data.dat \
--bleiTrainLabelFile data/predicting_funding/slda_in/kiva-train-label.dat \
--bleiTrainIdFile data/predicting_funding/slda_in/kiva-train-loan-ids.dat \
--bleiTestCorpusFile data/predicting_funding/slda_in/kiva-test-data.dat \
--bleiTestLabelFile data/predicting_funding/slda_in/kiva-test-label.dat \
--bleiTestIdFile data/predicting_funding/slda_in/kiva-test-loan-ids.dat \
--test_size 4000 \
--train_size 36000
Reading original labels from data/predicting_funding/kiva-labels-with-loan-ids.dat ... done number of original instances in class 0: 20130 number of original instances in class 1: 387467 trainSizePerClass = 18000 testSizePerClass = 2000 number of indices of class 0 into training set: 18000 number of indices of class 0 into test set: 2000 number of indices of class 1 into training set: 18000 number of indices of class 1 into test set: 2000
!/Users/frederik/install/slda-master/slda est \
data/predicting_funding/slda_in/kiva-train-data.dat \
data/predicting_funding/slda_in/kiva-train-label.dat \
data/predicting_funding/slda_settings.txt 1.0 20 \
seeded \
data/predicting_funding/slda_out
reading data from data/predicting_funding/slda_in/kiva-train-data.dat number of docs : 36000 number of terms : 1000 number of total words : 1586372 reading labels from data/predicting_funding/slda_in/kiva-train-label.dat number of classes : 2 alpha is fixed ... var max iter 20 var convergence 1.00E-03 em max iter 50 em convergence 1.00E-04 L2 penalty 1.00E-02 number of topics is 20 models will be saved in data/predicting_funding/slda_out initializing ... initialized with document 34178 initialized with document 10008 initialized with document 25433 initialized with document 9977 initialized with document 29749 initialized with document 32096 initialized with document 32855 initialized with document 22617 initialized with document 18778 initialized with document 11136 initialized with document 3205 initialized with document 11750 initialized with document 12212 initialized with document 17155 initialized with document 30234 initialized with document 21452 initialized with document 30654 initialized with document 25511 initialized with document 21017 initialized with document 9955 initialized with document 1732 initialized with document 26592 initialized with document 18184 initialized with document 13450 initialized with document 21135 initialized with document 22168 initialized with document 13417 initialized with document 18661 initialized with document 18072 initialized with document 29592 initialized with document 27412 initialized with document 5049 initialized with document 27639 initialized with document 10378 initialized with document 5565 initialized with document 3153 initialized with document 8424 initialized with document 11093 initialized with document 1258 initialized with document 29106 initialized with document 34160 initialized with document 18690 initialized with document 23417 initialized with document 33607 initialized with document 23151 initialized with document 28736 initialized with document 32353 initialized with document 18758 initialized with document 27358 initialized with document 6841 initialized with document 28581 initialized with document 35645 initialized with document 22256 initialized with document 16222 initialized with document 421 initialized with document 26222 initialized with document 6849 initialized with document 19089 initialized with document 3677 initialized with document 30533 initialized with document 2709 initialized with document 28944 initialized with document 28324 initialized with document 32259 initialized with document 2790 initialized with document 1848 initialized with document 23808 initialized with document 10760 initialized with document 12307 initialized with document 29928 initialized with document 33200 initialized with document 22623 initialized with document 23265 initialized with document 23132 initialized with document 4595 initialized with document 12306 initialized with document 24888 initialized with document 34387 initialized with document 6099 initialized with document 603 initialized with document 443 initialized with document 29701 initialized with document 3871 initialized with document 20509 initialized with document 18346 initialized with document 26437 initialized with document 21175 initialized with document 24433 initialized with document 9461 initialized with document 27419 initialized with document 20141 initialized with document 30914 initialized with document 27592 initialized with document 28428 initialized with document 9336 initialized with document 13861 initialized with document 14768 initialized with document 12416 initialized with document 33619 initialized with document 35400 initialized with document 8882 initialized with document 2308 initialized with document 11527 initialized with document 26795 initialized with document 15873 initialized with document 11170 initialized with document 29570 initialized with document 19512 initialized with document 19882 initialized with document 26647 initialized with document 23124 initialized with document 30024 initialized with document 8770 initialized with document 29340 initialized with document 26219 initialized with document 22165 initialized with document 28864 initialized with document 32715 initialized with document 1402 initialized with document 12867 initialized with document 22864 initialized with document 30512 initialized with document 887 initialized with document 22918 initialized with document 15830 initialized with document 3872 initialized with document 23321 initialized with document 25453 initialized with document 25618 initialized with document 31675 initialized with document 15828 initialized with document 4453 initialized with document 5319 initialized with document 27429 initialized with document 33573 initialized with document 1432 initialized with document 8905 initialized with document 23631 initialized with document 20148 initialized with document 31061 initialized with document 14837 initialized with document 12297 initialized with document 24237 initialized with document 14944 initialized with document 22200 initialized with document 29125 initialized with document 28509 initialized with document 31299 initialized with document 8974 initialized with document 7243 initialized with document 26564 initialized with document 20417 initialized with document 3304 initialized with document 1228 initialized with document 3080 initialized with document 34668 initialized with document 5785 initialized with document 7775 initialized with document 10751 initialized with document 21251 initialized with document 19888 initialized with document 15722 initialized with document 35989 initialized with document 1497 initialized with document 32426 initialized with document 17448 initialized with document 33568 initialized with document 7548 initialized with document 9172 initialized with document 12647 initialized with document 32129 initialized with document 148 initialized with document 10327 initialized with document 7632 initialized with document 4487 initialized with document 31793 initialized with document 24620 initialized with document 11071 initialized with document 27087 initialized with document 530 initialized with document 33559 initialized with document 19654 initialized with document 15491 initialized with document 33652 initialized with document 3490 initialized with document 25551 initialized with document 14323 initialized with document 19330 initialized with document 5101 initialized with document 25912 initialized with document 21720 initialized with document 23685 initialized with document 22744 initialized with document 17806 initialized with document 11422 initialized with document 23985 initialized with document 8472 initialized with document 4024 initialized with document 29056 initialized with document 32983 initialized with document 13828 initialized with document 21457 initialized with document 11370 initialized with document 16964 initialized with document 10708 initialized with document 1555 initialized with document 824 initialized with document 8765 initialized with document 27231 initialized with document 31675 initialized with document 17072 initialized with document 19396 initialized with document 21491 initialized with document 25807 initialized with document 33075 initialized with document 5580 initialized with document 26695 initialized with document 8199 initialized with document 28416 initialized with document 6916 initialized with document 33284 initialized with document 32731 initialized with document 7811 initialized with document 18855 initialized with document 9923 initialized with document 25759 initialized with document 27624 initialized with document 20134 initialized with document 14043 initialized with document 503 initialized with document 20658 initialized with document 26213 initialized with document 26279 initialized with document 4688 initialized with document 1801 initialized with document 34214 initialized with document 9825 initialized with document 14213 initialized with document 3300 initialized with document 20357 initialized with document 6032 initialized with document 27435 initialized with document 11286 initialized with document 12727 initialized with document 25305 initialized with document 8774 initialized with document 32771 initialized with document 21349 initialized with document 25438 initialized with document 34967 initialized with document 13108 initialized with document 22193 initialized with document 6638 initialized with document 21659 initialized with document 18022 initialized with document 21645 initialized with document 19759 initialized with document 13558 initialized with document 21122 initialized with document 19285 initialized with document 18886 initialized with document 1289 initialized with document 29429 initialized with document 23774 initialized with document 3137 initialized with document 26779 initialized with document 25707 initialized with document 32022 initialized with document 35478 initialized with document 17235 initialized with document 9798 initialized with document 22553 initialized with document 26161 initialized with document 144 initialized with document 7459 initialized with document 6361 initialized with document 35812 initialized with document 13210 initialized with document 19518 initialized with document 28366 initialized with document 20375 initialized with document 2590 initialized with document 27701 initialized with document 11729 initialized with document 13580 initialized with document 222 initialized with document 23669 initialized with document 7022 initialized with document 4625 initialized with document 6586 initialized with document 26107 initialized with document 20906 initialized with document 4104 initialized with document 16771 initialized with document 9444 initialized with document 13586 initialized with document 4763 initialized with document 28604 initialized with document 4823 initialized with document 31873 initialized with document 28990 initialized with document 18854 initialized with document 6516 initialized with document 23958 initialized with document 11789 initialized with document 12066 initialized with document 14420 initialized with document 31297 initialized with document 17968 initialized with document 28176 initialized with document 12215 initialized with document 16016 initialized with document 17411 initialized with document 23921 initialized with document 4732 initialized with document 17522 initialized with document 10293 initialized with document 20633 initialized with document 25925 initialized with document 34406 initialized with document 25849 initialized with document 956 initialized with document 27287 initialized with document 4335 initialized with document 4601 initialized with document 26112 initialized with document 14265 initialized with document 31607 initialized with document 19118 initialized with document 35584 initialized with document 2851 initialized with document 11155 initialized with document 26851 initialized with document 26531 initialized with document 16042 initialized with document 21226 initialized with document 4018 initialized with document 1941 initialized with document 30062 initialized with document 29725 initialized with document 28559 initialized with document 21646 initialized with document 34238 initialized with document 2878 initialized with document 17385 initialized with document 23552 initialized with document 10344 initialized with document 20141 initialized with document 1598 initialized with document 25201 initialized with document 9285 initialized with document 3703 initialized with document 20839 initialized with document 14377 initialized with document 17960 initialized with document 22679 initialized with document 34288 initialized with document 7467 initialized with document 20373 initialized with document 17556 initialized with document 5259 initialized with document 2790 initialized with document 13472 initialized with document 9279 initialized with document 26860 initialized with document 18123 initialized with document 286 initialized with document 14722 initialized with document 2958 initialized with document 3848 initialized with document 3218 initialized with document 25657 initialized with document 9626 initialized with document 7332 initialized with document 22562 initialized with document 32148 initialized with document 8510 initialized with document 6030 initialized with document 11612 initialized with document 4106 initialized with document 1910 initialized with document 22088 initialized with document 26406 initialized with document 6602 initialized with document 9296 initialized with document 16743 initialized with document 9562 initialized with document 30136 initialized with document 3653 initialized with document 13865 initialized with document 27979 initialized with document 10683 initialized with document 8733 initialized with document 14028 initialized with document 8847 initialized with document 16773 initialized with document 17722 initialized with document 13266 initialized with document 3922 initialized with document 21202 initialized with document 17084 initialized with document 4269 initialized with document 25460 initialized with document 16677 initialized with document 28335 initialized with document 25265 initialized with document 886 initialized with document 5303 initialized with document 23932 initialized with document 20502 initialized with document 17185 initialized with document 21757 initialized with document 25395 initialized with document 26586 initialized with document 18675 initialized with document 9492 initialized with document 5922 initialized with document 4215 initialized with document 10341 initialized with document 30774 initialized with document 10981 initialized with document 14934 initialized with document 5157 initialized with document 17412 initialized with document 10405 initialized with document 571 initialized with document 29352 initialized with document 26419 initialized with document 7067 initialized with document 13824 initialized with document 26091 initialized with document 22759 initialized with document 17878 initialized with document 14803 initialized with document 13654 initialized with document 15725 initialized with document 6555 initialized with document 31754 initialized with document 14477 initialized with document 6488 initialized with document 34215 initialized with document 21961 initialized with document 19270 initialized with document 17757 initialized with document 6975 initialized with document 31650 initialized with document 7926 initialized with document 9445 initialized with document 1159 initialized with document 19672 initialized with document 4749 initialized with document 22056 initialized with document 7516 initialized with document 3589 initialized with document 13690 initialized with document 14651 initialized with document 24593 initialized with document 3421 initialized with document 24365 initialized with document 14623 initialized with document 9541 initialized with document 5143 initialized with document 6773 initialized with document 33812 initialized with document 28092 initialized with document 13158 initialized with document 16594 initialized with document 10267 initialized with document 5402 initialized with document 16376 initialized with document 21795 initialized with document 15102 initialized with document 13649 initialized with document 26514 initialized with document 26484 initialized with document 20056 initialized with document 24323 initialized with document 663 initialized with document 264 initialized with document 30160 initialized with document 9535 initialized with document 19102 initialized with document 30900 initialized with document 19143 initialized with document 15210 initialized with document 17057 initialized with document 9554 initialized with document 28356 initialized with document 31207 initialized with document 12477 initialized with document 30375 initialized with document 4606 initialized with document 11796 initialized with document 18551 initialized with document 32087 initialized with document 14726 initialized with document 15543 initialized with document 3412 initialized with document 18663 initialized with document 25123 initialized with document 24811 initialized with document 29684 initialized with document 12762 initialized with document 34606 initialized with document 17716 initialized with document 23744 initialized with document 22283 initialized with document 28219 initialized with document 17575 initialized with document 25538 initialized with document 7828 initialized with document 12247 initialized with document 2877 initialized with document 8727 initialized with document 4078 initialized with document 8930 initialized with document 2343 initialized with document 3371 initialized with document 3214 initialized with document 35980 initialized with document 30828 initialized with document 20169 initialized with document 9924 initialized with document 6665 initialized with document 4522 initialized with document 34676 initialized with document 33771 initialized with document 34877 initialized with document 23107 initialized with document 7738 initialized with document 7213 initialized with document 12352 initialized with document 11052 initialized with document 34006 initialized with document 24973 initialized with document 19332 initialized with document 13839 initialized with document 5744 initialized with document 15807 initialized with document 13503 initialized with document 13945 initialized with document 12194 initialized with document 3 initialized with document 32371 initialized with document 15311 initialized with document 22889 initialized with document 23099 initialized with document 9015 initialized with document 17258 initialized with document 1894 initialized with document 33486 initialized with document 16890 initialized with document 2664 initialized with document 836 initialized with document 13288 initialized with document 3058 initialized with document 5987 initialized with document 35869 initialized with document 20559 initialized with document 19363 initialized with document 12581 initialized with document 25835 initialized with document 33725 initialized with document 35621 initialized with document 33198 initialized with document 24390 initialized with document 31621 initialized with document 21573 initialized with document 34718 initialized with document 33176 initialized with document 16291 initialized with document 19249 initialized with document 20675 initialized with document 35461 initialized with document 13445 initialized with document 9496 initialized with document 20970 initialized with document 17777 initialized with document 22693 initialized with document 27689 initialized with document 19296 initialized with document 17762 initialized with document 31 initialized with document 32947 initialized with document 18201 initialized with document 32671 initialized with document 26846 initialized with document 22810 initialized with document 35138 initialized with document 20029 initialized with document 12769 initialized with document 9915 initialized with document 157 initialized with document 31297 initialized with document 33466 initialized with document 5898 initialized with document 26861 initialized with document 33914 initialized with document 22889 initialized with document 10474 initialized with document 2967 initialized with document 12198 initialized with document 3553 initialized with document 19570 initialized with document 13138 initialized with document 34379 initialized with document 14969 initialized with document 1687 initialized with document 22454 initialized with document 27223 initialized with document 35236 initialized with document 12595 initialized with document 32495 initialized with document 31183 initialized with document 31563 initialized with document 3595 initialized with document 25700 initialized with document 23313 initialized with document 23546 initialized with document 35679 initialized with document 26361 initialized with document 12343 initialized with document 3170 initialized with document 29347 initialized with document 656 initialized with document 6005 initialized with document 2007 initialized with document 15191 initialized with document 28020 initialized with document 12883 initialized with document 3896 initialized with document 4368 initialized with document 18281 initialized with document 14403 initialized with document 3557 initialized with document 25534 initialized with document 1515 initialized with document 8417 initialized with document 6372 initialized with document 18460 initialized with document 30917 initialized with document 29030 initialized with document 1107 initialized with document 2330 initialized with document 21318 initialized with document 31886 initialized with document 942 initialized with document 34300 initialized with document 5586 initialized with document 7012 initialized with document 34659 initialized with document 2656 initialized with document 32866 initialized with document 14898 initialized with document 21580 initialized with document 32348 initialized with document 2927 initialized with document 9389 initialized with document 25718 initialized with document 11291 initialized with document 32097 initialized with document 23349 initialized with document 35456 initialized with document 26394 initialized with document 25987 initialized with document 26629 initialized with document 28639 initialized with document 13258 initialized with document 23279 initialized with document 34281 initialized with document 33283 initialized with document 27668 initialized with document 25594 initialized with document 23026 initialized with document 17186 initialized with document 28499 initialized with document 25703 initialized with document 10735 initialized with document 32416 initialized with document 18621 initialized with document 34156 initialized with document 18112 initialized with document 4774 initialized with document 14611 initialized with document 17925 initialized with document 12645 initialized with document 16750 initialized with document 11271 initialized with document 13770 initialized with document 22974 initialized with document 11493 initialized with document 35626 initialized with document 5085 initialized with document 35027 initialized with document 10294 initialized with document 10879 initialized with document 12336 initialized with document 24864 initialized with document 24330 initialized with document 21473 initialized with document 7757 initialized with document 33763 initialized with document 5730 initialized with document 26205 initialized with document 26078 initialized with document 17418 initialized with document 31491 initialized with document 20991 initialized with document 23708 initialized with document 31692 initialized with document 19238 initialized with document 661 initialized with document 3170 initialized with document 15191 initialized with document 10833 initialized with document 5762 initialized with document 17998 initialized with document 17323 initialized with document 6662 initialized with document 27944 initialized with document 18098 initialized with document 23093 initialized with document 27912 initialized with document 11510 initialized with document 30940 initialized with document 14511 initialized with document 7115 initialized with document 22969 initialized with document 7855 initialized with document 12444 initialized with document 824 initialized with document 4592 initialized with document 2740 initialized with document 22546 initialized with document 28034 initialized with document 13950 initialized with document 16453 initialized with document 15450 initialized with document 28431 initialized with document 9752 initialized with document 35894 initialized with document 10971 initialized with document 404 initialized with document 32465 initialized with document 22968 initialized with document 32592 initialized with document 14741 initialized with document 29798 initialized with document 3802 initialized with document 35666 initialized with document 35700 initialized with document 13000 initialized with document 29381 initialized with document 21240 initialized with document 4733 initialized with document 3118 initialized with document 10578 initialized with document 26660 initialized with document 5818 initialized with document 9851 initialized with document 32180 initialized with document 30785 initialized with document 5277 initialized with document 12842 initialized with document 16365 initialized with document 23024 initialized with document 2432 initialized with document 1413 initialized with document 20419 initialized with document 34615 initialized with document 35239 initialized with document 33627 initialized with document 9091 initialized with document 9527 initialized with document 26444 initialized with document 17743 initialized with document 32148 initialized with document 34495 initialized with document 256 initialized with document 16646 initialized with document 15159 initialized with document 931 initialized with document 32308 initialized with document 11305 initialized with document 6530 initialized with document 12291 initialized with document 26 initialized with document 15989 initialized with document 8572 initialized with document 1833 initialized with document 4127 initialized with document 11823 initialized with document 913 initialized with document 5512 initialized with document 14667 initialized with document 30352 initialized with document 22283 initialized with document 12154 initialized with document 6520 initialized with document 5474 initialized with document 16315 initialized with document 23541 initialized with document 32769 initialized with document 34565 initialized with document 23013 initialized with document 14455 initialized with document 33406 initialized with document 21531 initialized with document 11122 initialized with document 559 initialized with document 10342 initialized with document 8810 initialized with document 34066 initialized with document 23182 initialized with document 34929 initialized with document 35362 initialized with document 33034 initialized with document 30993 initialized with document 1975 initialized with document 12881 initialized with document 15484 initialized with document 27518 initialized with document 3559 initialized with document 22856 initialized with document 16463 initialized with document 30528 initialized with document 5161 initialized with document 5992 initialized with document 10336 initialized with document 13017 initialized with document 5175 initialized with document 33612 initialized with document 842 initialized with document 18430 initialized with document 20328 initialized with document 6152 initialized with document 23687 initialized with document 34959 initialized with document 27723 initialized with document 14055 initialized with document 1162 initialized with document 17193 initialized with document 17702 initialized with document 9001 initialized with document 30753 initialized with document 2683 initialized with document 25012 initialized with document 33984 initialized with document 29055 initialized with document 28525 initialized with document 26871 initialized with document 22834 initialized with document 24197 initialized with document 15833 initialized with document 4202 initialized with document 21865 initialized with document 18342 initialized with document 28295 initialized with document 35100 initialized with document 14914 initialized with document 27987 initialized with document 26469 initialized with document 5002 initialized with document 2017 initialized with document 25825 initialized with document 7614 initialized with document 21661 initialized with document 12482 initialized with document 20431 initialized with document 24745 initialized with document 27128 initialized with document 24569 initialized with document 9162 initialized with document 16628 initialized with document 26941 initialized with document 24029 initialized with document 32802 initialized with document 32249 initialized with document 18530 initialized with document 30305 initialized with document 2639 initialized with document 35420 initialized with document 19407 initialized with document 3615 initialized with document 30859 initialized with document 7430 initialized with document 14032 initialized with document 21143 initialized with document 25016 initialized with document 5915 initialized with document 33143 initialized with document 26638 initialized with document 24729 initialized with document 24330 initialized with document 20603 initialized with document 18728 initialized with document 33037 initialized with document 2556 initialized with document 7881 initialized with document 34240 initialized with document 18858 initialized with document 1532 initialized with document 20157 initialized with document 20759 initialized with document 13281 initialized with document 28418 initialized with document 19233 initialized with document 35173 initialized with document 18697 initialized with document 11458 initialized with document 31081 initialized with document 22588 initialized with document 35732 initialized with document 7614 initialized with document 2375 initialized with document 34338 initialized with document 14578 initialized with document 15007 initialized with document 34632 initialized with document 3278 initialized with document 29524 initialized with document 31449 initialized with document 4023 initialized with document 6283 initialized with document 860 initialized with document 22940 initialized with document 4683 initialized with document 17485 initialized with document 8498 initialized with document 27064 initialized with document 23684 initialized with document 11864 initialized with document 33989 initialized with document 10629 initialized with document 30320 initialized with document 24018 initialized with document 22696 initialized with document 24989 initialized with document 21863 initialized with document 3336 initialized with document 7017 initialized with document 30886 initialized with document 11817 initialized with document 31578 initialized with document 23483 initialized with document 17384 initialized with document 31240 initialized with document 18054 initialized with document 12457 initialized with document 16481 initialized with document 26085 initialized with document 7397 initialized with document 10788 initialized with document 23624 initialized with document 24026 initialized with document 21886 initialized with document 33888 initialized with document 12476 initialized with document 20484 initialized with document 11109 initialized with document 23184 initialized with document 12168 initialized with document 19655 initialized with document 6503 initialized with document 17232 initialized with document 23423 initialized with document 23302 initialized with document 15563 initialized with document 15865 initialized with document 29171 initialized with document 29702 initialized with document 10225 initialized with document 35217 initialized with document 5967 initialized with document 2644 initialized with document 23829 initialized with document 11718 initialized with document 26445 initialized with document 4878 initialized with document 32454 initialized with document 5321 initialized with document 28578 initialized with document 618 initialized with document 7069 initialized with document 10164 initialized with document 9743 initialized with document 30732 initialized with document 20059 initialized with document 32774 initialized with document 34523 initialized with document 10764 initialized with document 11777 initialized with document 505 initialized with document 24591 initialized with document 32018 initialized with document 1970 initialized with document 22034 initialized with document 22231 initialized with document 30173 **** em iteration 1 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -10679870.5855118856 **** m-step **** maximizing ... step: 00 -> f: -24951.935157 step: 10 -> f: -24882.951588 final f: -24882.942231 **** em iteration 2 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -10415813.8865509871 **** m-step **** maximizing ... step: 00 -> f: -24664.638515 step: 10 -> f: -24509.200383 step: 20 -> f: -24504.756611 step: 30 -> f: -24504.755957 step: 40 -> f: -24504.755939 final f: -24504.755937 **** em iteration 3 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -10360622.6886378806 **** m-step **** maximizing ... step: 00 -> f: -23676.576936 step: 10 -> f: -23399.587629 step: 20 -> f: -23399.051384 final f: -23399.051381 **** em iteration 4 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -10300358.9240206033 **** m-step **** maximizing ... step: 00 -> f: -21853.696743 step: 10 -> f: -21713.516891 step: 20 -> f: -21708.909446 step: 30 -> f: -21708.903980 step: 40 -> f: -21708.903962 final f: -21708.903962 **** em iteration 5 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -10222936.0970635787 **** m-step **** maximizing ... step: 00 -> f: -20826.492370 step: 10 -> f: -20754.924389 step: 20 -> f: -20747.943442 step: 30 -> f: -20747.932864 step: 40 -> f: -20747.932804 final f: -20747.932803 **** em iteration 6 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -10132007.3556194603 **** m-step **** maximizing ... step: 00 -> f: -20681.028669 step: 10 -> f: -20516.806766 step: 20 -> f: -20510.046833 step: 30 -> f: -20510.002599 step: 40 -> f: -20510.002579 final f: -20510.002578 **** em iteration 7 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -10046664.5151122063 **** m-step **** maximizing ... step: 00 -> f: -20583.245000 step: 10 -> f: -20484.469317 step: 20 -> f: -20481.800128 step: 30 -> f: -20480.456864 step: 40 -> f: -20480.453939 final f: -20480.453910 **** em iteration 8 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9980006.4182685092 **** m-step **** maximizing ... step: 00 -> f: -20608.001031 step: 10 -> f: -20565.224894 step: 20 -> f: -20564.663919 step: 30 -> f: -20564.662327 final f: -20564.662319 **** em iteration 9 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9931040.1757531520 **** m-step **** maximizing ... step: 00 -> f: -20704.496648 step: 10 -> f: -20687.218840 step: 20 -> f: -20686.954479 step: 30 -> f: -20686.954359 final f: -20686.954359 **** em iteration 10 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9894191.5112079680 **** m-step **** maximizing ... step: 00 -> f: -20813.849693 step: 10 -> f: -20805.580413 step: 20 -> f: -20805.569180 final f: -20805.569180 **** em iteration 11 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9865586.0220282413 **** m-step **** maximizing ... step: 00 -> f: -20907.238443 step: 10 -> f: -20902.243793 step: 20 -> f: -20902.204257 final f: -20902.204236 **** em iteration 12 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9843172.5703866854 **** m-step **** maximizing ... step: 00 -> f: -20982.591200 step: 10 -> f: -20979.220047 step: 20 -> f: -20979.216487 final f: -20979.216487 **** em iteration 13 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9825584.2302060444 **** m-step **** maximizing ... step: 00 -> f: -21041.665830 step: 10 -> f: -21039.658603 step: 20 -> f: -21039.468431 step: 30 -> f: -21039.468309 final f: -21039.468309 **** em iteration 14 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9811725.7615807075 **** m-step **** maximizing ... step: 00 -> f: -21089.185608 step: 10 -> f: -21087.797223 step: 20 -> f: -21087.793958 step: 30 -> f: -21087.793954 final f: -21087.793954 **** em iteration 15 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9800702.8309648540 **** m-step **** maximizing ... step: 00 -> f: -21127.876055 step: 10 -> f: -21126.941655 step: 20 -> f: -21126.940606 final f: -21126.940606 **** em iteration 16 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9791794.2687534280 **** m-step **** maximizing ... step: 00 -> f: -21160.510867 step: 10 -> f: -21159.799290 step: 20 -> f: -21159.797275 final f: -21159.797275 **** em iteration 17 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9784475.6504090037 **** m-step **** maximizing ... step: 00 -> f: -21188.285223 step: 10 -> f: -21187.727516 step: 20 -> f: -21187.721948 step: 30 -> f: -21187.721866 final f: -21187.721865 **** em iteration 18 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9778393.6675054412 **** m-step **** maximizing ... step: 00 -> f: -21211.301282 step: 10 -> f: -21210.836346 step: 20 -> f: -21210.835271 final f: -21210.835270 **** em iteration 19 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9773283.3429162577 **** m-step **** maximizing ... step: 00 -> f: -21231.425682 step: 10 -> f: -21231.015387 step: 20 -> f: -21231.015010 final f: -21231.015010 **** em iteration 20 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9768930.8417494111 **** m-step **** maximizing ... step: 00 -> f: -21248.583271 step: 10 -> f: -21248.212039 step: 20 -> f: -21248.211935 final f: -21248.211935 **** em iteration 21 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9765171.1877001319 **** m-step **** maximizing ... step: 00 -> f: -21263.824546 step: 10 -> f: -21263.481502 step: 20 -> f: -21263.481483 final f: -21263.481483 **** em iteration 22 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9761879.8790301103 **** m-step **** maximizing ... step: 00 -> f: -21277.497388 step: 10 -> f: -21277.169630 step: 20 -> f: -21277.169344 final f: -21277.169344 **** em iteration 23 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9758968.5723861624 **** m-step **** maximizing ... step: 00 -> f: -21289.912992 step: 10 -> f: -21289.586410 step: 20 -> f: -21289.586386 final f: -21289.586386 **** em iteration 24 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9756368.8245841004 **** m-step **** maximizing ... step: 00 -> f: -21301.479392 step: 10 -> f: -21301.174758 final f: -21301.174717 **** em iteration 25 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9754030.9333271924 **** m-step **** maximizing ... step: 00 -> f: -21312.316467 step: 10 -> f: -21312.024190 step: 20 -> f: -21312.023988 final f: -21312.023988 **** em iteration 26 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9751914.0669772215 **** m-step **** maximizing ... step: 00 -> f: -21322.619449 step: 10 -> f: -21322.348489 step: 20 -> f: -21322.348164 final f: -21322.348164 **** em iteration 27 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9749984.8418760542 **** m-step **** maximizing ... step: 00 -> f: -21332.884599 step: 10 -> f: -21332.636433 step: 20 -> f: -21332.636308 final f: -21332.636308 **** em iteration 28 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9748214.7274887767 **** m-step **** maximizing ... step: 00 -> f: -21343.114612 step: 10 -> f: -21342.891124 final f: -21342.891064 **** em iteration 29 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9746583.4529125933 **** m-step **** maximizing ... step: 00 -> f: -21353.231975 step: 10 -> f: -21353.034601 step: 20 -> f: -21353.034480 final f: -21353.034480 **** em iteration 30 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9745071.3569002599 **** m-step **** maximizing ... step: 00 -> f: -21363.846931 step: 10 -> f: -21363.676109 step: 20 -> f: -21363.675975 final f: -21363.675975 **** em iteration 31 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9743666.9535370879 **** m-step **** maximizing ... step: 00 -> f: -21374.574301 step: 10 -> f: -21374.434267 step: 20 -> f: -21374.434253 final f: -21374.434253 **** em iteration 32 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9742360.5591412764 **** m-step **** maximizing ... step: 00 -> f: -21385.328955 step: 10 -> f: -21385.217946 step: 20 -> f: -21385.217872 final f: -21385.217871 **** em iteration 33 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9741144.7788030300 **** m-step **** maximizing ... step: 00 -> f: -21396.011002 step: 10 -> f: -21395.925519 final f: -21395.925371 **** em iteration 34 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9740011.2911324203 **** m-step **** maximizing ... step: 00 -> f: -21406.356566 step: 10 -> f: -21406.290914 step: 20 -> f: -21406.290868 final f: -21406.290868 **** em iteration 35 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9738952.6252822187 **** m-step **** maximizing ... step: 00 -> f: -21416.298585 step: 10 -> f: -21416.250034 final f: -21416.250032 **** em iteration 36 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9737961.7968516890 **** m-step **** maximizing ... step: 00 -> f: -21425.715752 step: 10 -> f: -21425.679528 step: 20 -> f: -21425.679449 final f: -21425.679449 **** em iteration 37 **** **** e-step **** document 0 document 1000 document 2000 document 3000 document 4000 document 5000 document 6000 document 7000 document 8000 document 9000 document 10000 document 11000 document 12000 document 13000 document 14000 document 15000 document 16000 document 17000 document 18000 document 19000 document 20000 document 21000 document 22000 document 23000 document 24000 document 25000 document 26000 document 27000 document 28000 document 29000 document 30000 document 31000 document 32000 document 33000 document 34000 document 35000 likelihood: -9737028.6692849211 **** m-step **** maximizing ... step: 00 -> f: -21434.475475 step: 10 -> f: -21434.447372 step: 20 -> f: -21434.447371 final f: -21434.447371 final e step document 0 final e step document 1000 final e step document 2000 final e step document 3000 final e step document 4000 final e step document 5000 final e step document 6000 final e step document 7000 final e step document 8000 final e step document 9000 final e step document 10000 final e step document 11000 final e step document 12000 final e step document 13000 final e step document 14000 final e step document 15000 final e step document 16000 final e step document 17000 final e step document 18000 final e step document 19000 final e step document 20000 final e step document 21000 final e step document 22000 final e step document 23000 final e step document 24000 final e step document 25000 final e step document 26000 final e step document 27000 final e step document 28000 final e step document 29000 final e step document 30000 final e step document 31000 final e step document 32000 final e step document 33000 final e step document 34000 final e step document 35000
!/Users/frederik/install/slda-master/slda inf \
data/predicting_funding/slda_in/kiva-test-data.dat \
data/predicting_funding/slda_in/kiva-test-label.dat \
data/predicting_funding/slda_settings.txt \
data/predicting_funding/slda_out/final.model \
data/predicting_funding/slda_out
reading data from data/predicting_funding/slda_in/kiva-test-data.dat number of docs : 4000 number of terms : 1000 number of total words : 175601 reading labels from data/predicting_funding/slda_in/kiva-test-label.dat number of classes : 2 alpha is fixed ... var max iter 20 var convergence 1.00E-03 em max iter 50 em convergence 1.00E-04 L2 penalty 1.00E-02 results will be saved in data/predicting_funding/slda_out document 0 document 1000 document 2000 document 3000 average accuracy: 0.686
!python src/evaluate_sdla.py --predictedFile data/predicting_funding/slda_out/inf-labels.dat \
--expectedFile data/predicting_funding/slda_in/kiva-test-label.dat \
--average weighted
Reading predicted classes from data/predicting_funding/slda_out/inf-labels.dat ... done Reading expected classes from data/predicting_funding/slda_in/kiva-test-label.dat ... done (0.70696378830083562, 0.63449999999999995, 0.66877470355731217, None)
!python src/print_topics.py --vocabFile data/predicting_funding/kiva.lda-c.vocab \
--sldaModelFile data/predicting_funding/slda_out/final.model.text
Loading vocabulary from data/predicting_funding/kiva.lda-c.vocab ... done ETA TOPIC -4.56 : [ income earn farming rice living hopes main per day two ] -2.89 : [ school food water sugar selling married oil six four rice ] -2.87 : [ store php additional future nwtf requested general save earns philippines ] -2.61 : [ city time works day making located well says bank work ] -2.42 : [ community meet every help household week expenses clients daily fellowship ] -1.70 : [ needs sell market support financial provide opportunity price higher however ] -1.25 : [ use five married hopes goal future profits plans previous kes ] -0.82 : [ women members sells profit drinks grocery various made cosmetics live ] -0.69 : [ school mother education pay father young year single parents daughter ] +0.55 : [ husband woman like married man would money family hardworking purchase ] +0.70 : [ one group fund farmers season also harvest farmer year savings ] +0.95 : [ milk cattle also livestock farm raising two purchase pigs cows ] +1.15 : [ able work support thanks help continue service lenders little due ] +1.16 : [ improve living since requesting old partner services married experience car ] +1.18 : [ small help used town expand requested large hopes brac lives ] +1.81 : [ better work life corn supplies give crops land worked basic ] +1.83 : [ family income new conditions local start cover purchase credit lives ] +2.11 : [ products customers store good many quality sales life offer household ] +2.71 : [ selling increase shop capital clothing started sells clothes ago working ] +2.97 : [ house home lives old son wife vegetables materials live requesting ]
!python src/loan_funding_predictor_poc.py --trainSldaGammaFile data/predicting_funding/slda_out/final.gamma \
--trainLabelFile data/predicting_funding/slda_in/kiva-train-label.dat \
--trainIdFile data/predicting_funding/slda_in/kiva-train-loan-ids.dat \
--testSldaGammaFile data/predicting_funding/slda_out/inf-gamma.dat \
--testLabelFile data/predicting_funding/slda_out/inf-labels.dat \
--testIdFile data/predicting_funding/slda_in/kiva-test-loan-ids.dat \
--feat "slda" \
--feat "borrower_majority_gender" \
--feat "loan_amount" \
--feat "has_image" \
--feat "posted_day_of_month" \
--feat "posted_month" \
--feat "geo_lat" \
--feat "geo_lon" \
--feat "repayment_term" \
--feat "nr_borrowers" \
--feat "bonus_credit_eligibility" \
--feat "translated" \
--feat "partner_rating" \
--feat "partner_delinquency_rate" \
--feat "partner_loans_posted" \
--feat "partner_total_amount_raised" \
--feat "en_description_length" \
--feat "constant" \
Retrieving Kiva partner list from http://api.kivaws.org/v1/partners.json ... parsing it ... done {u'average_loan_size_percent_per_capita_income': 13.3, u'charges_fees_and_interest': True, u'countries': [{u'iso_code': u'JO', u'location': {u'geo': {u'level': u'country', u'pairs': u'31 36', u'type': u'point'}}, u'name': u'Jordan', u'region': u'Middle East'}], u'currency_exchange_loss_rate': 0, u'default_rate': 0.70201247428837, u'default_rate_note': u'', u'delinquency_rate': 0.53427779934297, u'delinquency_rate_note': u'', u'id': 185, u'image': {u'id': 790950, u'template_id': 1}, u'loans_at_risk_rate': 2.8314480394342, u'loans_posted': 4355, u'name': u'National Microfinance Bank', u'portfolio_yield': 38, u'portfolio_yield_note': u'', u'profitability': 11, u'rating': u'3.5', u'social_performance_strengths': [{u'description': u'The work of most microfinance institutions helps to combat poverty, but these Field Partners do even more.', u'id': 1, u'name': u'Anti-Poverty Focus'}, {u'description': u'These Field Partners embrace technology and innovation to better address the needs of the people they serve.', u'id': 7, u'name': u'Innovation'}], u'start_date': u'2011-03-30T22:00:05Z', u'status': u'active', u'total_amount_raised': 4596300, u'url': u'http://www.nmb.com.jo'} Reading sLDA gamma values from data/predicting_funding/slda_out/final.gamma ... done Reading loan IDs from data/predicting_funding/slda_in/kiva-train-loan-ids.dat ... done Creating MongoDB cursor to collect 36000 loan instances by ID ... done Storing loan documents in an associative list ... done Reading labels from data/predicting_funding/slda_in/kiva-train-label.dat ... done Reading sLDA gamma values from data/predicting_funding/slda_out/inf-gamma.dat ... done Reading loan IDs from data/predicting_funding/slda_in/kiva-test-loan-ids.dat ... done Creating MongoDB cursor to collect 4000 loan instances by ID ... done Storing loan documents in an associative list ... done Reading labels from data/predicting_funding/slda_out/inf-labels.dat ... done (1/10) Building logres model for C value 1.000000000000000 ... applying it ... done (2/10) Building logres model for C value 10.000000000000000 ... applying it ... done (3/10) Building logres model for C value 100.000000000000000 ... applying it ... done (4/10) Building logres model for C value 1000.000000000000000 ... applying it ... done (5/10) Building logres model for C value 10000.000000000000000 ... applying it ... done (6/10) Building logres model for C value 100000.000000000000000 ... applying it ... done (7/10) Building logres model for C value 1000000.000000000000000 ... applying it ... done (8/10) Building logres model for C value 10000000.000000000000000 ... applying it ... done (9/10) Building logres model for C value 100000000.000000000000000 ... applying it ... done (10/10) Building logres model for C value 1000000000.000000000000000 ... applying it ... done scoreList = [(0.88479630730623371, 0.0048239057208544428), (0.8847947570138327, 0.0048241677073719283), (0.88479473208185144, 0.0048247744944466412), (0.88479483209083243, 0.0048250525723827582), (0.88479490708829867, 0.0048250350114758551), (0.88479493208835491, 0.0048250425021032816), (0.88479493208835491, 0.0048250425021032816), (0.88479493208835491, 0.0048250425021032816), (0.88479493208835491, 0.0048250425021032816), (0.88479493208835491, 0.0048250425021032816)] best logres model has: roc_auc score: 88.48% (+/- 0.48%) C value: 1 log10(C) value: 0.0
This version allows to save the final logres model on disk, so it can be used in real time by a Kiva Loan Funding predictor running as a web service. Another major requirement was to isolate the (quite hefty) data preprocessing / feature generation in separate modules, so it can be applied to unseen instances on the fly.
!python src/loan_funding_predictor.py --startYear 2012 \
--endYear 2014 \
--logResModelFile data/predicting_funding/logres_out/kivaLoanFundingPredictor.pkl
Creating MongoDB cursor to collect instances from 2012 through 2014 ... done Storing loan documents in KivaLoans instance ... read 10000 documents ... read 20000 documents ... read 30000 documents ... read 40000 documents ... read 50000 documents ... read 60000 documents ... read 70000 documents ... read 80000 documents ... read 90000 documents ... read 100000 documents ... read 110000 documents ... read 120000 documents ... read 130000 documents ... read 140000 documents ... read 150000 documents ... read 160000 documents ... read 170000 documents ... read 180000 documents ... read 190000 documents ... read 200000 documents ... read 210000 documents ... read 220000 documents ... read 230000 documents ... read 240000 documents ... read 250000 documents ... read 260000 documents ... read 270000 documents ... read 280000 documents ... read 290000 documents ... read 300000 documents ... read 310000 documents ... read 320000 documents ... read 330000 documents ... read 340000 documents ... read 350000 documents ... read 360000 documents ... read 370000 documents ... read 380000 documents ... read 390000 documents ... read 400000 documents ... read 410000 documents ... read 420000 documents ... read 430000 documents ... read 440000 documents ... done Getting ground truth labels ... done number of 0s (not fully funded) = 21212 number of 1s (fully funded) = 419913 Setting up sLDA feature generator ... done Getting all features at once ... done Building training DataFrame ... done columns = ['Baseline', 'BonusCreditEligibility', 'GeoLatitude', 'GeoLongitude', 'HasImage', 'HasTranslator', 'Log10EnglishDescriptionLength', 'Log10LoanAmount', 'Log10NumberOfBorrowers', 'MajorityGender', 'PostedMonthApr', 'PostedMonthAug', 'PostedMonthDec', 'PostedMonthFeb', 'PostedMonthJan', 'PostedMonthJul', 'PostedMonthJun', 'PostedMonthMar', 'PostedMonthMay', 'PostedMonthNov', 'PostedMonthOct', 'PostedMonthSep', 'RepaymentTerm', 'topic_00', 'topic_01', 'topic_02', 'topic_03', 'topic_04', 'topic_05', 'topic_06', 'topic_07', 'topic_08', 'topic_09', 'topic_10', 'topic_11', 'topic_12', 'topic_13', 'topic_14', 'topic_15', 'topic_16', 'topic_17', 'topic_18', 'topic_19', 'DelinquencyRate', 'LoansPosted', 'Rating', 'TotalAmountRaised'] trainDF.head(3) = Baseline BonusCreditEligibility GeoLatitude GeoLongitude HasImage \ 377304 1 0 8.500000 -11.500000 1 377251 1 1 15.500000 47.500000 1 361766 1 1 13.833333 -88.916667 1 HasTranslator Log10EnglishDescriptionLength Log10LoanAmount \ 377304 1 2.033424 2.977724 377251 1 2.354108 3.138303 361766 1 2.431364 2.698970 Log10NumberOfBorrowers MajorityGender ... topic_14 \ 377304 0 0 ... 2.094038 377251 0 1 ... 7.586307 361766 0 1 ... 2.619955 377304 4.643855 1.040619 1.046571 3.371768 2.226809 17.891291 377251 3.346344 11.254259 1.242014 3.563978 5.165154 0.246900 361766 2.244133 9.694625 22.782238 5.915603 1.877627 3.917693 LoansPosted Rating TotalAmountRaised 377304 3531 2.0 2886600 377251 3551 3.5 2901950 361766 7162 3.5 4998150 [3 rows x 47 columns] Feeding data to the predictor .../usr/local/lib/python2.7/site-packages/sklearn/utils/validation.py:332: UserWarning: StandardScaler assumes floating point values as input, got object "got %s" % (estimator, X.dtype)) done (1/20) Building logres model for C value 0.000000000100000 ... applying it ... done (2/20) Building logres model for C value 0.000000001000000 ... applying it ... done (3/20) Building logres model for C value 0.000000010000000 ... applying it ... done (4/20) Building logres model for C value 0.000000100000000 ... applying it ... done (5/20) Building logres model for C value 0.000001000000000 ... applying it ... done (6/20) Building logres model for C value 0.000010000000000 ... applying it ... done (7/20) Building logres model for C value 0.000100000000000 ... applying it ... done (8/20) Building logres model for C value 0.001000000000000 ... applying it ... done (9/20) Building logres model for C value 0.010000000000000 ... applying it ... done (10/20) Building logres model for C value 0.100000000000000 ... applying it ... done (11/20) Building logres model for C value 1.000000000000000 ... applying it ... done (12/20) Building logres model for C value 10.000000000000000 ... applying it ... done (13/20) Building logres model for C value 100.000000000000000 ... applying it ... done (14/20) Building logres model for C value 1000.000000000000000 ... applying it ... done (15/20) Building logres model for C value 10000.000000000000000 ... applying it ... done (16/20) Building logres model for C value 100000.000000000000000 ... applying it ... done (17/20) Building logres model for C value 1000000.000000000000000 ... applying it ... done (18/20) Building logres model for C value 10000000.000000000000000 ... applying it ... done (19/20) Building logres model for C value 100000000.000000000000000 ... applying it ... done (20/20) Building logres model for C value 1000000000.000000000000000 ... applying it ... done best logres model has: roc_auc score: 50.00% (+/- 0.00%) C value: 1e-10 log10(C) value: -10.0 config [['Baseline']] : (0.5, 0.0) (1/20) Building logres model for C value 0.000000000100000 ... applying it ... done (2/20) Building logres model for C value 0.000000001000000 ... applying it ... done (3/20) Building logres model for C value 0.000000010000000 ... applying it ... done (4/20) Building logres model for C value 0.000000100000000 ... applying it ... done (5/20) Building logres model for C value 0.000001000000000 ... applying it ... done (6/20) Building logres model for C value 0.000010000000000 ... applying it ... done (7/20) Building logres model for C value 0.000100000000000 ... applying it ... done (8/20) Building logres model for C value 0.001000000000000 ... applying it ... done (9/20) Building logres model for C value 0.010000000000000 ... applying it ... done (10/20) Building logres model for C value 0.100000000000000 ... applying it ... done (11/20) Building logres model for C value 1.000000000000000 ... applying it ... done (12/20) Building logres model for C value 10.000000000000000 ... applying it ... done (13/20) Building logres model for C value 100.000000000000000 ... applying it ... done (14/20) Building logres model for C value 1000.000000000000000 ... applying it ... done (15/20) Building logres model for C value 10000.000000000000000 ... applying it ... done (16/20) Building logres model for C value 100000.000000000000000 ... applying it ... done (17/20) Building logres model for C value 1000000.000000000000000 ... applying it ... done (18/20) Building logres model for C value 10000000.000000000000000 ... applying it ... done (19/20) Building logres model for C value 100000000.000000000000000 ... applying it ... done (20/20) Building logres model for C value 1000000000.000000000000000 ... applying it ... done best logres model has: roc_auc score: 76.50% (+/- 0.27%) C value: 1e-09 log10(C) value: -9.0 config [['Baseline'], ['Log10LoanAmount']] : (0.76497708058841862, 0.0026845918868675426) BETTER RESULT! (1/20) Building logres model for C value 0.000000000100000 ... applying it ... done (2/20) Building logres model for C value 0.000000001000000 ... applying it ... done (3/20) Building logres model for C value 0.000000010000000 ... applying it ... done (4/20) Building logres model for C value 0.000000100000000 ... applying it ... done (5/20) Building logres model for C value 0.000001000000000 ... applying it ... done (6/20) Building logres model for C value 0.000010000000000 ... applying it ... done (7/20) Building logres model for C value 0.000100000000000 ... applying it ... done (8/20) Building logres model for C value 0.001000000000000 ... applying it ... done (9/20) Building logres model for C value 0.010000000000000 ... applying it ... done (10/20) Building logres model for C value 0.100000000000000 ... applying it ... done (11/20) Building logres model for C value 1.000000000000000 ... applying it ... done (12/20) Building logres model for C value 10.000000000000000 ... applying it ... done (13/20) Building logres model for C value 100.000000000000000 ... applying it ... done (14/20) Building logres model for C value 1000.000000000000000 ... applying it ... done (15/20) Building logres model for C value 10000.000000000000000 ... applying it ... done (16/20) Building logres model for C value 100000.000000000000000 ... applying it ... done (17/20) Building logres model for C value 1000000.000000000000000 ... applying it ... done (18/20) Building logres model for C value 10000000.000000000000000 ... applying it ... done (19/20) Building logres model for C value 100000000.000000000000000 ... applying it ... done (20/20) Building logres model for C value 1000000000.000000000000000 ... applying it ... done best logres model has: roc_auc score: 80.72% (+/- 0.29%) C value: 10 log10(C) value: 1.0 config [['Baseline'], ['Log10LoanAmount'], ['topic_00', 'topic_01', 'topic_02', 'topic_03', 'topic_04', 'topic_05', 'topic_06', 'topic_07', 'topic_08', 'topic_09', 'topic_10', 'topic_11', 'topic_12', 'topic_13', 'topic_14', 'topic_15', 'topic_16', 'topic_17', 'topic_18', 'topic_19']] : (0.80724149857527971, 0.0029114021612315125) BETTER RESULT! (1/20) Building logres model for C value 0.000000000100000 ... applying it ... done (2/20) Building logres model for C value 0.000000001000000 ... applying it ... done (3/20) Building logres model for C value 0.000000010000000 ... applying it ... done (4/20) Building logres model for C value 0.000000100000000 ... applying it ... done (5/20) Building logres model for C value 0.000001000000000 ... applying it ... done (6/20) Building logres model for C value 0.000010000000000 ... applying it ... done (7/20) Building logres model for C value 0.000100000000000 ... applying it ... done (8/20) Building logres model for C value 0.001000000000000 ... applying it ... done (9/20) Building logres model for C value 0.010000000000000 ... applying it ... done (10/20) Building logres model for C value 0.100000000000000 ... applying it ... done (11/20) Building logres model for C value 1.000000000000000 ... applying it ... done (12/20) Building logres model for C value 10.000000000000000 ... applying it ... done (13/20) Building logres model for C value 100.000000000000000 ... applying it ... done (14/20) Building logres model for C value 1000.000000000000000 ... applying it ... done (15/20) Building logres model for C value 10000.000000000000000 ... applying it ... done (16/20) Building logres model for C value 100000.000000000000000 ... applying it ... done (17/20) Building logres model for C value 1000000.000000000000000 ... applying it ... done (18/20) Building logres model for C value 10000000.000000000000000 ... applying it ... done (19/20) Building logres model for C value 100000000.000000000000000 ... applying it ... done (20/20) Building logres model for C value 1000000000.000000000000000 ... applying it ... done best logres model has: roc_auc score: 83.55% (+/- 0.38%) C value: 0.01 log10(C) value: -2.0 config [['Baseline'], ['Log10LoanAmount'], ['topic_00', 'topic_01', 'topic_02', 'topic_03', 'topic_04', 'topic_05', 'topic_06', 'topic_07', 'topic_08', 'topic_09', 'topic_10', 'topic_11', 'topic_12', 'topic_13', 'topic_14', 'topic_15', 'topic_16', 'topic_17', 'topic_18', 'topic_19'], ['PostedMonthJan', 'PostedMonthFeb', 'PostedMonthMar', 'PostedMonthApr', 'PostedMonthMay', 'PostedMonthJun', 'PostedMonthJul', 'PostedMonthAug', 'PostedMonthSep', 'PostedMonthOct', 'PostedMonthNov', 'PostedMonthDec']] : (0.83551111862577054, 0.0038379630008923155) BETTER RESULT! (1/20) Building logres model for C value 0.000000000100000 ... applying it ... done (2/20) Building logres model for C value 0.000000001000000 ... applying it ... done (3/20) Building logres model for C value 0.000000010000000 ... applying it ... done (4/20) Building logres model for C value 0.000000100000000 ... applying it ... done (5/20) Building logres model for C value 0.000001000000000 ... applying it ... done (6/20) Building logres model for C value 0.000010000000000 ... applying it ... done (7/20) Building logres model for C value 0.000100000000000 ... applying it ... done (8/20) Building logres model for C value 0.001000000000000 ... applying it ... done (9/20) Building logres model for C value 0.010000000000000 ... applying it ... done (10/20) Building logres model for C value 0.100000000000000 ... applying it ... done (11/20) Building logres model for C value 1.000000000000000 ... applying it ... done (12/20) Building logres model for C value 10.000000000000000 ... applying it ... done (13/20) Building logres model for C value 100.000000000000000 ... applying it ... done (14/20) Building logres model for C value 1000.000000000000000 ... applying it ... done (15/20) Building logres model for C value 10000.000000000000000 ... applying it ... done (16/20) Building logres model for C value 100000.000000000000000 ... applying it ... done (17/20) Building logres model for C value 1000000.000000000000000 ... applying it ... done (18/20) Building logres model for C value 10000000.000000000000000 ... applying it ... done (19/20) Building logres model for C value 100000000.000000000000000 ... applying it ... done (20/20) Building logres model for C value 1000000000.000000000000000 ... applying it ... done best logres model has: roc_auc score: 85.99% (+/- 0.24%) C value: 0.01 log10(C) value: -2.0 config [['Baseline'], ['Log10LoanAmount'], ['topic_00', 'topic_01', 'topic_02', 'topic_03', 'topic_04', 'topic_05', 'topic_06', 'topic_07', 'topic_08', 'topic_09', 'topic_10', 'topic_11', 'topic_12', 'topic_13', 'topic_14', 'topic_15', 'topic_16', 'topic_17', 'topic_18', 'topic_19'], ['PostedMonthJan', 'PostedMonthFeb', 'PostedMonthMar', 'PostedMonthApr', 'PostedMonthMay', 'PostedMonthJun', 'PostedMonthJul', 'PostedMonthAug', 'PostedMonthSep', 'PostedMonthOct', 'PostedMonthNov', 'PostedMonthDec'], ['MajorityGender']] : (0.8598600212264278, 0.0023718889092303025) BETTER RESULT! (1/20) Building logres model for C value 0.000000000100000 ... applying it ... done (2/20) Building logres model for C value 0.000000001000000 ... applying it ... done (3/20) Building logres model for C value 0.000000010000000 ... applying it ... done (4/20) Building logres model for C value 0.000000100000000 ... applying it ... done (5/20) Building logres model for C value 0.000001000000000 ... applying it ... done (6/20) Building logres model for C value 0.000010000000000 ... applying it ... done (7/20) Building logres model for C value 0.000100000000000 ... applying it ... done (8/20) Building logres model for C value 0.001000000000000 ... applying it ... done (9/20) Building logres model for C value 0.010000000000000 ... applying it ... done (10/20) Building logres model for C value 0.100000000000000 ... applying it ... done (11/20) Building logres model for C value 1.000000000000000 ... applying it ... done (12/20) Building logres model for C value 10.000000000000000 ... applying it ... done (13/20) Building logres model for C value 100.000000000000000 ... applying it ... done (14/20) Building logres model for C value 1000.000000000000000 ... applying it ... done (15/20) Building logres model for C value 10000.000000000000000 ... applying it ... done (16/20) Building logres model for C value 100000.000000000000000 ... applying it ... done (17/20) Building logres model for C value 1000000.000000000000000 ... applying it ... done (18/20) Building logres model for C value 10000000.000000000000000 ... applying it ... done (19/20) Building logres model for C value 100000000.000000000000000 ... applying it ... done (20/20) Building logres model for C value 1000000000.000000000000000 ... applying it ... done best logres model has: roc_auc score: 87.13% (+/- 0.22%) C value: 100 log10(C) value: 2.0 config [['Baseline'], ['Log10LoanAmount'], ['topic_00', 'topic_01', 'topic_02', 'topic_03', 'topic_04', 'topic_05', 'topic_06', 'topic_07', 'topic_08', 'topic_09', 'topic_10', 'topic_11', 'topic_12', 'topic_13', 'topic_14', 'topic_15', 'topic_16', 'topic_17', 'topic_18', 'topic_19'], ['PostedMonthJan', 'PostedMonthFeb', 'PostedMonthMar', 'PostedMonthApr', 'PostedMonthMay', 'PostedMonthJun', 'PostedMonthJul', 'PostedMonthAug', 'PostedMonthSep', 'PostedMonthOct', 'PostedMonthNov', 'PostedMonthDec'], ['MajorityGender'], ['Log10NumberOfBorrowers']] : (0.87129700209852068, 0.0021549010302392125) BETTER RESULT! (1/20) Building logres model for C value 0.000000000100000 ... applying it ... done (2/20) Building logres model for C value 0.000000001000000 ... applying it ... done (3/20) Building logres model for C value 0.000000010000000 ... applying it ... done (4/20) Building logres model for C value 0.000000100000000 ... applying it ... done (5/20) Building logres model for C value 0.000001000000000 ... applying it ... done (6/20) Building logres model for C value 0.000010000000000 ... applying it ... done (7/20) Building logres model for C value 0.000100000000000 ... applying it ... done (8/20) Building logres model for C value 0.001000000000000 ... applying it ... done (9/20) Building logres model for C value 0.010000000000000 ... applying it ... done (10/20) Building logres model for C value 0.100000000000000 ... applying it ... done (11/20) Building logres model for C value 1.000000000000000 ... applying it ... done (12/20) Building logres model for C value 10.000000000000000 ... applying it ... done (13/20) Building logres model for C value 100.000000000000000 ... applying it ... done (14/20) Building logres model for C value 1000.000000000000000 ... applying it ... done (15/20) Building logres model for C value 10000.000000000000000 ... applying it ... done (16/20) Building logres model for C value 100000.000000000000000 ... applying it ... done (17/20) Building logres model for C value 1000000.000000000000000 ... applying it ... done (18/20) Building logres model for C value 10000000.000000000000000 ... applying it ... done (19/20) Building logres model for C value 100000000.000000000000000 ... applying it ... done (20/20) Building logres model for C value 1000000000.000000000000000 ... applying it ... done best logres model has: roc_auc score: 87.42% (+/- 0.20%) C value: 100 log10(C) value: 2.0 config [['Baseline'], ['Log10LoanAmount'], ['topic_00', 'topic_01', 'topic_02', 'topic_03', 'topic_04', 'topic_05', 'topic_06', 'topic_07', 'topic_08', 'topic_09', 'topic_10', 'topic_11', 'topic_12', 'topic_13', 'topic_14', 'topic_15', 'topic_16', 'topic_17', 'topic_18', 'topic_19'], ['PostedMonthJan', 'PostedMonthFeb', 'PostedMonthMar', 'PostedMonthApr', 'PostedMonthMay', 'PostedMonthJun', 'PostedMonthJul', 'PostedMonthAug', 'PostedMonthSep', 'PostedMonthOct', 'PostedMonthNov', 'PostedMonthDec'], ['MajorityGender'], ['Log10NumberOfBorrowers'], ['LoansPosted', 'TotalAmountRaised']] : (0.87416503723221817, 0.0020017434076329624) BETTER RESULT! (1/20) Building logres model for C value 0.000000000100000 ... applying it ... done (2/20) Building logres model for C value 0.000000001000000 ... applying it ... done (3/20) Building logres model for C value 0.000000010000000 ... applying it ... done (4/20) Building logres model for C value 0.000000100000000 ... applying it ... done (5/20) Building logres model for C value 0.000001000000000 ... applying it ... done (6/20) Building logres model for C value 0.000010000000000 ... applying it ... done (7/20) Building logres model for C value 0.000100000000000 ... applying it ... done (8/20) Building logres model for C value 0.001000000000000 ... applying it ... done (9/20) Building logres model for C value 0.010000000000000 ... applying it ... done (10/20) Building logres model for C value 0.100000000000000 ... applying it ... done (11/20) Building logres model for C value 1.000000000000000 ... applying it ... done (12/20) Building logres model for C value 10.000000000000000 ... applying it ... done (13/20) Building logres model for C value 100.000000000000000 ... applying it ... done (14/20) Building logres model for C value 1000.000000000000000 ... applying it ... done (15/20) Building logres model for C value 10000.000000000000000 ... applying it ... done (16/20) Building logres model for C value 100000.000000000000000 ... applying it ... done (17/20) Building logres model for C value 1000000.000000000000000 ... applying it ... done (18/20) Building logres model for C value 10000000.000000000000000 ... applying it ... done (19/20) Building logres model for C value 100000000.000000000000000 ... applying it ... done (20/20) Building logres model for C value 1000000000.000000000000000 ... applying it ... done best logres model has: roc_auc score: 87.63% (+/- 0.26%) C value: 10 log10(C) value: 1.0 config [['Baseline'], ['Log10LoanAmount'], ['topic_00', 'topic_01', 'topic_02', 'topic_03', 'topic_04', 'topic_05', 'topic_06', 'topic_07', 'topic_08', 'topic_09', 'topic_10', 'topic_11', 'topic_12', 'topic_13', 'topic_14', 'topic_15', 'topic_16', 'topic_17', 'topic_18', 'topic_19'], ['PostedMonthJan', 'PostedMonthFeb', 'PostedMonthMar', 'PostedMonthApr', 'PostedMonthMay', 'PostedMonthJun', 'PostedMonthJul', 'PostedMonthAug', 'PostedMonthSep', 'PostedMonthOct', 'PostedMonthNov', 'PostedMonthDec'], ['MajorityGender'], ['Log10NumberOfBorrowers'], ['LoansPosted', 'TotalAmountRaised'], ['GeoLatitude', 'GeoLongitude']] : (0.87627831607141471, 0.0025550878988913699) BETTER RESULT! (1/20) Building logres model for C value 0.000000000100000 ... applying it ... done (2/20) Building logres model for C value 0.000000001000000 ... applying it ... done (3/20) Building logres model for C value 0.000000010000000 ... applying it ... done (4/20) Building logres model for C value 0.000000100000000 ... applying it ... done (5/20) Building logres model for C value 0.000001000000000 ... applying it ... done (6/20) Building logres model for C value 0.000010000000000 ... applying it ... done (7/20) Building logres model for C value 0.000100000000000 ... applying it ... done (8/20) Building logres model for C value 0.001000000000000 ... applying it ... done (9/20) Building logres model for C value 0.010000000000000 ... applying it ... done (10/20) Building logres model for C value 0.100000000000000 ... applying it ... done (11/20) Building logres model for C value 1.000000000000000 ... applying it ... done (12/20) Building logres model for C value 10.000000000000000 ... applying it ... done (13/20) Building logres model for C value 100.000000000000000 ... applying it ... done (14/20) Building logres model for C value 1000.000000000000000 ... applying it ... done (15/20) Building logres model for C value 10000.000000000000000 ... applying it ... done (16/20) Building logres model for C value 100000.000000000000000 ... applying it ... done (17/20) Building logres model for C value 1000000.000000000000000 ... applying it ... done (18/20) Building logres model for C value 10000000.000000000000000 ... applying it ... done (19/20) Building logres model for C value 100000000.000000000000000 ... applying it ... done (20/20) Building logres model for C value 1000000000.000000000000000 ... applying it ... done best logres model has: roc_auc score: 87.77% (+/- 0.26%) C value: 10000 log10(C) value: 4.0 config [['Baseline'], ['Log10LoanAmount'], ['topic_00', 'topic_01', 'topic_02', 'topic_03', 'topic_04', 'topic_05', 'topic_06', 'topic_07', 'topic_08', 'topic_09', 'topic_10', 'topic_11', 'topic_12', 'topic_13', 'topic_14', 'topic_15', 'topic_16', 'topic_17', 'topic_18', 'topic_19'], ['PostedMonthJan', 'PostedMonthFeb', 'PostedMonthMar', 'PostedMonthApr', 'PostedMonthMay', 'PostedMonthJun', 'PostedMonthJul', 'PostedMonthAug', 'PostedMonthSep', 'PostedMonthOct', 'PostedMonthNov', 'PostedMonthDec'], ['MajorityGender'], ['Log10NumberOfBorrowers'], ['LoansPosted', 'TotalAmountRaised'], ['GeoLatitude', 'GeoLongitude'], ['RepaymentTerm']] : (0.87774961786777261, 0.0026330676733586152) BETTER RESULT! (1/20) Building logres model for C value 0.000000000100000 ... applying it ... done (2/20) Building logres model for C value 0.000000001000000 ... applying it ... done (3/20) Building logres model for C value 0.000000010000000 ... applying it ... done (4/20) Building logres model for C value 0.000000100000000 ... applying it ... done (5/20) Building logres model for C value 0.000001000000000 ... applying it ... done (6/20) Building logres model for C value 0.000010000000000 ... applying it ... done (7/20) Building logres model for C value 0.000100000000000 ... applying it ... done (8/20) Building logres model for C value 0.001000000000000 ... applying it ... done (9/20) Building logres model for C value 0.010000000000000 ... applying it ... done (10/20) Building logres model for C value 0.100000000000000 ... applying it ... done (11/20) Building logres model for C value 1.000000000000000 ... applying it ... done (12/20) Building logres model for C value 10.000000000000000 ... applying it ... done (13/20) Building logres model for C value 100.000000000000000 ... applying it ... done (14/20) Building logres model for C value 1000.000000000000000 ... applying it ... done (15/20) Building logres model for C value 10000.000000000000000 ... applying it ... done (16/20) Building logres model for C value 100000.000000000000000 ... applying it ... done (17/20) Building logres model for C value 1000000.000000000000000 ... applying it ... done (18/20) Building logres model for C value 10000000.000000000000000 ... applying it ... done (19/20) Building logres model for C value 100000000.000000000000000 ... applying it ... done (20/20) Building logres model for C value 1000000000.000000000000000 ... applying it ... done best logres model has: roc_auc score: 87.77% (+/- 0.27%) C value: 10000 log10(C) value: 4.0 config [['Baseline'], ['Log10LoanAmount'], ['topic_00', 'topic_01', 'topic_02', 'topic_03', 'topic_04', 'topic_05', 'topic_06', 'topic_07', 'topic_08', 'topic_09', 'topic_10', 'topic_11', 'topic_12', 'topic_13', 'topic_14', 'topic_15', 'topic_16', 'topic_17', 'topic_18', 'topic_19'], ['PostedMonthJan', 'PostedMonthFeb', 'PostedMonthMar', 'PostedMonthApr', 'PostedMonthMay', 'PostedMonthJun', 'PostedMonthJul', 'PostedMonthAug', 'PostedMonthSep', 'PostedMonthOct', 'PostedMonthNov', 'PostedMonthDec'], ['MajorityGender'], ['Log10NumberOfBorrowers'], ['LoansPosted', 'TotalAmountRaised'], ['GeoLatitude', 'GeoLongitude'], ['RepaymentTerm'], ['DelinquencyRate', 'Rating']] : (0.87768934190653647, 0.0027331616389033014) (1/20) Building logres model for C value 0.000000000100000 ... applying it ... done (2/20) Building logres model for C value 0.000000001000000 ... applying it ... done (3/20) Building logres model for C value 0.000000010000000 ... applying it ... done (4/20) Building logres model for C value 0.000000100000000 ... applying it ... done (5/20) Building logres model for C value 0.000001000000000 ... applying it ... done (6/20) Building logres model for C value 0.000010000000000 ... applying it ... done (7/20) Building logres model for C value 0.000100000000000 ... applying it ... done (8/20) Building logres model for C value 0.001000000000000 ... applying it ... done (9/20) Building logres model for C value 0.010000000000000 ... applying it ... done (10/20) Building logres model for C value 0.100000000000000 ... applying it ... done (11/20) Building logres model for C value 1.000000000000000 ... applying it ... done (12/20) Building logres model for C value 10.000000000000000 ... applying it ... done (13/20) Building logres model for C value 100.000000000000000 ... applying it ... done (14/20) Building logres model for C value 1000.000000000000000 ... applying it ... done (15/20) Building logres model for C value 10000.000000000000000 ... applying it ... done (16/20) Building logres model for C value 100000.000000000000000 ... applying it ... done (17/20) Building logres model for C value 1000000.000000000000000 ... applying it ... done (18/20) Building logres model for C value 10000000.000000000000000 ... applying it ... done (19/20) Building logres model for C value 100000000.000000000000000 ... applying it ... done (20/20) Building logres model for C value 1000000000.000000000000000 ... applying it ... done best logres model has: roc_auc score: 87.82% (+/- 0.31%) C value: 1000 log10(C) value: 3.0 config [['Baseline'], ['Log10LoanAmount'], ['topic_00', 'topic_01', 'topic_02', 'topic_03', 'topic_04', 'topic_05', 'topic_06', 'topic_07', 'topic_08', 'topic_09', 'topic_10', 'topic_11', 'topic_12', 'topic_13', 'topic_14', 'topic_15', 'topic_16', 'topic_17', 'topic_18', 'topic_19'], ['PostedMonthJan', 'PostedMonthFeb', 'PostedMonthMar', 'PostedMonthApr', 'PostedMonthMay', 'PostedMonthJun', 'PostedMonthJul', 'PostedMonthAug', 'PostedMonthSep', 'PostedMonthOct', 'PostedMonthNov', 'PostedMonthDec'], ['MajorityGender'], ['Log10NumberOfBorrowers'], ['LoansPosted', 'TotalAmountRaised'], ['GeoLatitude', 'GeoLongitude'], ['RepaymentTerm'], ['DelinquencyRate', 'Rating'], ['BonusCreditEligibility']] : (0.87819724370619201, 0.0031405240396688685) BETTER RESULT! (1/20) Building logres model for C value 0.000000000100000 ... applying it ... done (2/20) Building logres model for C value 0.000000001000000 ... applying it ... done (3/20) Building logres model for C value 0.000000010000000 ... applying it ... done (4/20) Building logres model for C value 0.000000100000000 ... applying it ... done (5/20) Building logres model for C value 0.000001000000000 ... applying it ... done (6/20) Building logres model for C value 0.000010000000000 ... applying it ... done (7/20) Building logres model for C value 0.000100000000000 ... applying it ... done (8/20) Building logres model for C value 0.001000000000000 ... applying it ... done (9/20) Building logres model for C value 0.010000000000000 ... applying it ... done (10/20) Building logres model for C value 0.100000000000000 ... applying it ... done (11/20) Building logres model for C value 1.000000000000000 ... applying it ... done (12/20) Building logres model for C value 10.000000000000000 ... applying it ... done (13/20) Building logres model for C value 100.000000000000000 ... applying it ... done (14/20) Building logres model for C value 1000.000000000000000 ... applying it ... done (15/20) Building logres model for C value 10000.000000000000000 ... applying it ... done (16/20) Building logres model for C value 100000.000000000000000 ... applying it ... done (17/20) Building logres model for C value 1000000.000000000000000 ... applying it ... done (18/20) Building logres model for C value 10000000.000000000000000 ... applying it ... done (19/20) Building logres model for C value 100000000.000000000000000 ... applying it ... done (20/20) Building logres model for C value 1000000000.000000000000000 ... applying it ... done best logres model has: roc_auc score: 87.81% (+/- 0.25%) C value: 100 log10(C) value: 2.0 config [['Baseline'], ['Log10LoanAmount'], ['topic_00', 'topic_01', 'topic_02', 'topic_03', 'topic_04', 'topic_05', 'topic_06', 'topic_07', 'topic_08', 'topic_09', 'topic_10', 'topic_11', 'topic_12', 'topic_13', 'topic_14', 'topic_15', 'topic_16', 'topic_17', 'topic_18', 'topic_19'], ['PostedMonthJan', 'PostedMonthFeb', 'PostedMonthMar', 'PostedMonthApr', 'PostedMonthMay', 'PostedMonthJun', 'PostedMonthJul', 'PostedMonthAug', 'PostedMonthSep', 'PostedMonthOct', 'PostedMonthNov', 'PostedMonthDec'], ['MajorityGender'], ['Log10NumberOfBorrowers'], ['LoansPosted', 'TotalAmountRaised'], ['GeoLatitude', 'GeoLongitude'], ['RepaymentTerm'], ['DelinquencyRate', 'Rating'], ['BonusCreditEligibility'], ['Log10EnglishDescriptionLength']] : (0.87808241597881176, 0.0024926082275705335) bestResult = 0.878197243706 bestConfig = [['Baseline'], ['Log10LoanAmount'], ['topic_00', 'topic_01', 'topic_02', 'topic_03', 'topic_04', 'topic_05', 'topic_06', 'topic_07', 'topic_08', 'topic_09', 'topic_10', 'topic_11', 'topic_12', 'topic_13', 'topic_14', 'topic_15', 'topic_16', 'topic_17', 'topic_18', 'topic_19'], ['PostedMonthJan', 'PostedMonthFeb', 'PostedMonthMar', 'PostedMonthApr', 'PostedMonthMay', 'PostedMonthJun', 'PostedMonthJul', 'PostedMonthAug', 'PostedMonthSep', 'PostedMonthOct', 'PostedMonthNov', 'PostedMonthDec'], ['MajorityGender'], ['Log10NumberOfBorrowers'], ['LoansPosted', 'TotalAmountRaised'], ['GeoLatitude', 'GeoLongitude'], ['RepaymentTerm'], ['DelinquencyRate', 'Rating'], ['BonusCreditEligibility'], ['Log10EnglishDescriptionLength']] Saving best classifier to disk ... done Preparing unseen data ... done Loading classifier from disk ... done unseenDF.head(3) = Baseline BonusCreditEligibility GeoLatitude GeoLongitude HasImage \ 847571 1 1 -2 30 1 847570 1 1 2 33 1 848289 1 0 4 -72 1 HasTranslator Log10EnglishDescriptionLength Log10LoanAmount \ 847571 1 2.187521 3.295567 847570 1 2.220108 3.197281 848289 1 2.167317 3.088136 Log10NumberOfBorrowers MajorityGender ... topic_14 \ 847571 0 1 ... 1.253125 847570 0 1 ... 2.872730 848289 0 1 ... 5.075311 topic_15 topic_16 topic_17 topic_18 topic_19 DelinquencyRate \ 847571 1.239835 1.016679 1.346673 1.212425 1.087503 0.000000 847570 1.360059 1.529400 4.831612 6.177116 3.094922 10.046096 848289 1.066923 10.297114 1.901048 1.970634 1.217917 3.157753 LoansPosted Rating TotalAmountRaised 847571 7203 4.0 7051875 847570 2867 3.5 2886900 848289 8367 3.5 6876525 [3 rows x 47 columns] probaList = [[ 0.09530087 0.90469913] [ 0.32417722 0.67582278] [ 0.64328611 0.35671389]] predictions = [1 1 0] sorted feature coefficients in trained model: ('TotalAmountRaised', 0.51512673372977458) ('PostedMonthMar', 0.49587882878259132) ('Log10NumberOfBorrowers', 0.48883803701379364) ('PostedMonthFeb', 0.42799728441252655) ('GeoLongitude', 0.38565914705597004) ('PostedMonthJan', 0.35079067702901506) ('topic_11', 0.34463381927852504) ['income', 'earn', 'farming', 'rice', 'living', 'hopes', 'main', 'per', 'day', 'two'] ('topic_13', 0.28708132847232587) ['city', 'time', 'works', 'day', 'making', 'located', 'well', 'says', 'bank', 'work'] ('topic_18', 0.21475821305783774) ['school', 'food', 'water', 'sugar', 'selling', 'married', 'oil', 'six', 'four', 'rice'] ('topic_09', 0.20934126188765559) ['school', 'mother', 'education', 'pay', 'father', 'young', 'year', 'single', 'parents', 'daughter'] ('topic_05', 0.20686539023306405) ['needs', 'sell', 'market', 'support', 'financial', 'provide', 'opportunity', 'price', 'higher', 'however'] ('topic_19', 0.15099384642581043) ['community', 'meet', 'every', 'help', 'household', 'week', 'expenses', 'clients', 'daily', 'fellowship'] ('topic_04', 0.078019280568203023) ['use', 'five', 'married', 'hopes', 'goal', 'future', 'profits', 'plans', 'previous', 'kes'] ('Rating', 0.077958867518446537) ('Log10EnglishDescriptionLength', 0.076418455975725913) ('topic_14', 0.061570138754807865) ['family', 'income', 'new', 'conditions', 'local', 'start', 'cover', 'purchase', 'credit', 'lives'] ('topic_17', 0.042947945859326486) ['better', 'work', 'life', 'corn', 'supplies', 'give', 'crops', 'land', 'worked', 'basic'] ('topic_03', 0.023512600959147277) ['milk', 'cattle', 'also', 'livestock', 'farm', 'raising', 'two', 'purchase', 'pigs', 'cows'] ('topic_15', 0.019221226724594188) ['store', 'php', 'additional', 'future', 'nwtf', 'requested', 'general', 'save', 'earns', 'philippines'] ('PostedMonthApr', 0.012681107688343872) ('DelinquencyRate', 0.0099085728485939755) ('topic_08', 0.0070316234981432591) ['able', 'work', 'support', 'thanks', 'help', 'continue', 'service', 'lenders', 'little', 'due'] ('Baseline', 0.0) ('topic_10', -0.026241133761063538) ['improve', 'living', 'since', 'requesting', 'old', 'partner', 'services', 'married', 'experience', 'car'] ('PostedMonthOct', -0.042498684080964508) ('topic_01', -0.071793214438988479) ['small', 'help', 'used', 'town', 'expand', 'requested', 'large', 'hopes', 'brac', 'lives'] ('GeoLatitude', -0.071888473425193139) ('topic_07', -0.078008030184428076) ['women', 'members', 'sells', 'profit', 'drinks', 'grocery', 'various', 'made', 'cosmetics', 'live'] ('topic_06', -0.10954977745880268) ['husband', 'woman', 'like', 'married', 'man', 'would', 'money', 'family', 'hardworking', 'purchase'] ('PostedMonthNov', -0.11373050680868292) ('PostedMonthAug', -0.12381247043109402) ('PostedMonthJul', -0.12545649415847018) ('topic_00', -0.14755071028433084) ['one', 'group', 'fund', 'farmers', 'season', 'also', 'harvest', 'farmer', 'year', 'savings'] ('topic_02', -0.1481791870952158) ['selling', 'increase', 'shop', 'capital', 'clothing', 'started', 'sells', 'clothes', 'ago', 'working'] ('PostedMonthMay', -0.14998063936421158) ('BonusCreditEligibility', -0.16710379963027269) ('topic_12', -0.16987378700739811) ['house', 'home', 'lives', 'old', 'son', 'wife', 'vegetables', 'materials', 'live', 'requesting'] ('PostedMonthDec', -0.17337927676927412) ('topic_16', -0.17389029153368829) ['products', 'customers', 'store', 'good', 'many', 'quality', 'sales', 'life', 'offer', 'household'] ('PostedMonthSep', -0.20777319104077704) ('PostedMonthJun', -0.28331282420885068) ('RepaymentTerm', -0.28978714414735179) ('LoansPosted', -0.53589567168998353) ('MajorityGender', -0.584374830570077) ('Log10LoanAmount', -1.4442547937444266)
Manual creation of PostedMonth contribution graph (need to go quick)
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
fig = plt.figure(20, figsize=(8,8))
months = ['Mar', 'Feb', 'Jan', 'Apr', 'Oct', 'Nov', 'Aug', 'Jul', 'May', 'Dec', 'Sep', 'Jun']
logresCoefficients = [0.49, 0.42, 0.35, 0.12, -0.04, -0.11, -0.12, -0.12, -0.15, -0.17, -0.20, -0.28]
index = np.arange(12)
bar_width = 0.50
plt.bar(index, logresCoefficients, bar_width)
plt.xlabel('Month', fontsize="20")
plt.ylabel('Impact on positive outcome (full funding) ', fontsize="20")
plt.title('LogRes Model Coefficients for PostedMonth features', fontsize="20", y=1.03)
plt.xticks(index + bar_width, months)
plt.setp(plt.gca().get_yticklabels(), fontsize="20")
plt.setp(plt.gca().get_xticklabels(), fontsize="20", ha="center", rotation=45)
#plt.autoscale(tight=True)
plt.tight_layout()
plt.show()
Manual creation of topic contribution graph (need to go quick)
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
fig = plt.figure(20, figsize=(20,8))
ax = fig.add_subplot(111)
topics = [['one', 'group', 'fund', 'farmers', 'season', 'also', 'harvest', 'farmer', 'year', 'savings'],
['small', 'help', 'used', 'town', 'expand', 'requested', 'large', 'hopes', 'brac', 'lives'],
['selling', 'increase', 'shop', 'capital', 'clothing', 'started', 'sells', 'clothes', 'ago', 'working'],
['milk', 'cattle', 'also', 'livestock', 'farm', 'raising', 'two', 'purchase', 'pigs', 'cows'],
['use', 'five', 'married', 'hopes', 'goal', 'future', 'profits', 'plans', 'previous', 'kes'],
['needs', 'sell', 'market', 'support', 'financial', 'provide', 'opportunity', 'price', 'higher', 'however'],
['husband', 'woman', 'like', 'married', 'man', 'would', 'money', 'family', 'hardworking', 'purchase'],
['women', 'members', 'sells', 'profit', 'drinks', 'grocery', 'various', 'made', 'cosmetics', 'live'],
['able', 'work', 'support', 'thanks', 'help', 'continue', 'service', 'lenders', 'little', 'due'],
['school', 'mother', 'education', 'pay', 'father', 'young', 'year', 'single', 'parents', 'daughter'],
['improve', 'living', 'since', 'requesting', 'old', 'partner', 'services', 'married', 'experience', 'car'],
['income', 'earn', 'farming', 'rice', 'living', 'hopes', 'main', 'per', 'day', 'two'],
['house', 'home', 'lives', 'old', 'son', 'wife', 'vegetables', 'materials', 'live', 'requesting'],
['city', 'time', 'works', 'day', 'making', 'located', 'well', 'says', 'bank', 'work'],
['family', 'income', 'new', 'conditions', 'local', 'start', 'cover', 'purchase', 'credit', 'lives'],
['store', 'php', 'additional', 'future', 'nwtf', 'requested', 'general', 'save', 'earns', 'philippines'],
['products', 'customers', 'store', 'good', 'many', 'quality', 'sales', 'life', 'offer', 'household'],
['better', 'work', 'life', 'corn', 'supplies', 'give', 'crops', 'land', 'worked', 'basic'],
['school', 'food', 'water', 'sugar', 'selling', 'married', 'oil', 'six', 'four', 'rice'],
['community', 'meet', 'every', 'help', 'household', 'week', 'expenses', 'clients', 'daily', 'fellowship']]
topicIndicesAndCoefficients = [
(11, 0.34463381927852504),
(13, 0.28708132847232587),
(18, 0.21475821305783774),
(9, 0.20934126188765559),
(5, 0.20686539023306405),
(19, 0.15099384642581043),
(4, 0.078019280568203023),
(14, 0.061570138754807865),
(17, 0.042947945859326486),
(3, 0.023512600959147277),
(15, 0.019221226724594188),
(8, 0.0070316234981432591),
(10, -0.026241133761063538) ,
(1, -0.071793214438988479) ,
(7, -0.078008030184428076),
(6, -0.10954977745880268) ,
(0, -0.14755071028433084),
(2, -0.1481791870952158) ,
(12, -0.16987378700739811) ,
(16, -0.17389029153368829)]
Xtopics = ["topic_%d" % (topicIndicesAndCoefficients[i][0]) for i in range(len(topics))]
topicLogresCoefficients = [topicIndicesAndCoefficients[i][1] for i in range(len(topics))]
index = np.arange(len(topics))
bar_width = 0.70
plt.bar(index, topicLogresCoefficients, bar_width)
plt.xlabel('Topic', fontsize="20")
plt.ylabel('Impact on positive outcome (full funding) ', fontsize="20")
plt.title('LogRes Model Coefficients for topical features', fontsize="20", y=1.03)
plt.xticks(index + bar_width, Xtopics)
for i in range(len(topics)):
coeff = topicLogresCoefficients[i]
string = "\n".join(topics[i])
if coeff >= 0:
y = -0.13
else:
y = 0.02
ax.annotate(string, xy=(index[i]+0.05, y), xytext=(index[i]+0.05, y), fontsize="10")
plt.setp(plt.gca().get_yticklabels(), fontsize="20")
plt.setp(plt.gca().get_xticklabels(), fontsize="10", ha="right", rotation=0)
plt.autoscale(tight=True)
plt.tight_layout()
plt.show()
Manual creation of other features' contribution graph (need to go quick)
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
fig = plt.figure(20, figsize=(12,8))
rankedFeaturesAndCoeffs = [
('TotalAmountRaised', 0.51512673372977458),
('Log10NumberOfBorrowers', 0.48883803701379364),
('GeoLongitude', 0.38565914705597004),
('Rating', 0.077958867518446537),
('Log10EnglishDescriptionLength', 0.076418455975725913),
('DelinquencyRate', 0.0099085728485939755),
('GeoLatitude', -0.071888473425193139),
('BonusCreditEligibility', -0.16710379963027269),
('RepaymentTerm', -0.28978714414735179),
('LoansPosted', -0.53589567168998353),
('MajorityGender', -0.584374830570077),
('Log10LoanAmount', -1.4442547937444266)]
features = ["%s" % (rankedFeaturesAndCoeffs[i][0]) for i in range(len(rankedFeaturesAndCoeffs))]
print features
logresCoeffs = [rankedFeaturesAndCoeffs[i][1] for i in range(len(rankedFeaturesAndCoeffs))]
print logresCoeffs
index = np.arange(len(rankedFeaturesAndCoeffs))
bar_width = 0.50
plt.bar(index, logresCoeffs, bar_width)
plt.xlabel('Other features', fontsize="20")
plt.ylabel('Impact on positive outcome\n(full funding) ', fontsize="20")
plt.title('LogRes Model Coefficients for remaining features', fontsize="20", y=1.03)
plt.xticks(index + bar_width, features)
plt.setp(plt.gca().get_yticklabels(), fontsize="20")
plt.setp(plt.gca().get_xticklabels(), fontsize="16", ha="right", rotation=45)
plt.autoscale(tight=True)
plt.tight_layout()
plt.show()
['TotalAmountRaised', 'Log10NumberOfBorrowers', 'GeoLongitude', 'Rating', 'Log10EnglishDescriptionLength', 'DelinquencyRate', 'GeoLatitude', 'BonusCreditEligibility', 'RepaymentTerm', 'LoansPosted', 'MajorityGender', 'Log10LoanAmount'] [0.5151267337297746, 0.48883803701379364, 0.38565914705597004, 0.07795886751844654, 0.07641845597572591, 0.009908572848593976, -0.07188847342519314, -0.1671037996302727, -0.2897871441473518, -0.5358956716899835, -0.584374830570077, -1.4442547937444266]