Data Science Paper Studies

Studies by Ashok to Confirm Numbers in Paper

Code

#=======================================================
# INITIALIZE
#=======================================================

from khipu import Khipu
(khipu_dict, all_khipus) = kamayuq.fetch_khipus()

# Initialize plotly
plotly.offline.init_notebook_mode(connected = False);

1. Khipus With Zero Cord Clusters

Code

khipus_with_zero_cord_clusters = []
for khipu in all_khipus:
    for cluster in khipu.cord_clusters():
        if cluster.num_cords() == 0:
            khipus_with_zero_cord_clusters.append(khipu.name())
malformed_khipus = sorted(list(set(khipus_with_zero_cord_clusters)))
print(f"Khipus with_zero cord clusters ({len(malformed_khipus)}): {malformed_khipus}")

Code

num_zero_khipus = 0
zero_cord_khipus = []
for aKhipu in all_khipus:
    is_zero = all([aCord.knotted_value()==0 for aCord in aKhipu.cc_cords()])
    if is_zero:
        num_zero_khipus += 1
        zero_cord_khipus.append(aKhipu.name())
zero_cord_khipus = kq.sort_khipus_by_name(zero_cord_khipus)
print(f"Found {num_zero_khipus} zero cord khipus:")
khipu_rep = "     '" + "'\n     '".join(zero_cord_khipus)
print(khipu_rep)

Found 11 zero cord khipus:
     'AS025'
     'HP025'
     'HP026'
     'HP028'
     'HP048'
     'QU001'
     'UR070'
     'UR071'
     'UR082'
     'UR158'
     'UR179

2. Top Cord, Sum Top Cord, and Double Sum Top Cord Distribution

2.1 Top Cord, Sum Top Cords, and Double Sum Top Cord Distributions

Lets look at how top cords, sum top cords (top cords which are sums) and double sum top cords distribute. Note that all double sum top cord khipus exist on khipus that also have “regular” top sum cords.

Code

from fieldmark_khipu_summary import Fieldmark_TopCords
topcord_Fieldmark = Fieldmark_TopCords()
topcord_df = topcord_Fieldmark.dataframes[0].dataframe
topcord_khipus = (len(topcord_Fieldmark.matching_khipus()))
num_top_cords = sum(topcord_df['num_top_cords'].values.tolist())
print(f"Top Cords Fieldmark: {topcord_khipus} khipus, {num_top_cords} total top cords")

from fieldmark_sum_top_cords import Fieldmark_Sum_Top_Cords
sum_topcord_Fieldmark = Fieldmark_Sum_Top_Cords()
sum_topcord_df = sum_topcord_Fieldmark.dataframes[0].dataframe
sum_topcord_khipus = (len(sum_topcord_Fieldmark.matching_khipus()))
num_sum_top_cords = sum(sum_topcord_df['num_sum_top_cords'].values.tolist())
print(f"Sum Top Cords Fieldmark: {sum_topcord_khipus} khipus, {num_sum_top_cords} total sum top cords")

from fieldmark_double_sum_top_cords import Fieldmark_Double_Sum_Top_Cords
double_sum_topcord_Fieldmark = Fieldmark_Double_Sum_Top_Cords()
double_sum_topcord_df = double_sum_topcord_Fieldmark.dataframes[0].dataframe
double_sum_topcord_khipus = (len(double_sum_topcord_Fieldmark.matching_khipus()))
double_num_sum_top_cords = sum(double_sum_topcord_df['num_double_sum_top_cords'].values.tolist())
print(f"Double Sum Top Cords Fieldmark: {double_sum_topcord_khipus} khipus, {double_num_sum_top_cords} total sum top cords")

print(f"\nSum Top Cord Khipus: {sorted(sum_topcord_Fieldmark.matching_khipus())}")
print(f"Double Sum Top Cord Khipus: {sorted(double_sum_topcord_Fieldmark.matching_khipus())}\n")

all_sum_topcord_khipus = set.union(set(sum_topcord_Fieldmark.matching_khipus()), set(double_sum_topcord_Fieldmark.matching_khipus()))
num_all_sum_topcord_khipus = len(all_sum_topcord_khipus)
num_all_sum_topcords = num_sum_top_cords + double_num_sum_top_cords

print(f"# of Khipus that have Sum Top Cords or Double Sum Top Cords: {num_all_sum_topcord_khipus} ")
percentage = round(100.*float(num_all_sum_topcords)/float(num_top_cords))
print(f"# of Sum Top Cords({num_sum_top_cords}) or Double Sum Top Cords({double_num_sum_top_cords}) as % of Top Cords({num_top_cords}) = {percentage}%")

Top Cords Fieldmark: 47 khipus, 376 total top cords
Sum Top Cords Fieldmark: 29 khipus, 86 total sum top cords
Double Sum Top Cords Fieldmark: 10 khipus, 21 total sum top cords

Sum Top Cord Khipus: ['AS001', 'AS002', 'AS003', 'AS004', 'AS006', 'AS007', 'AS044', 'AS066', 'AS072', 'AS199', 'AS215', 'HP047', 'QU009', 'QU014', 'QU015', 'QU016', 'QU019', 'UR047', 'UR048', 'UR087', 'UR1057', 'UR1096', 'UR118', 'UR119', 'UR149', 'UR166', 'UR186', 'UR241', 'UR246']
Double Sum Top Cord Khipus: ['AS044', 'AS066', 'AS072', 'HP047', 'QU016', 'UR087', 'UR1057', 'UR1096', 'UR186', 'UR241']

# of Khipus that have Sum Top Cords or Double Sum Top Cords: 29 
# of Sum Top Cords(86) or Double Sum Top Cords(21) as % of Top Cords(376) = 28%

Code

from fieldmark_sum_top_cords import Fieldmark_Sum_Top_Cords
aFieldmark = Fieldmark_Sum_Top_Cords()
fieldmark_dataframe = aFieldmark.dataframes[0].dataframe
print(f"A total of {sum(list(fieldmark_dataframe.num_sum_top_cords.values))} sum top cords exist over {len(aFieldmark.matching_khipus())} khipus with sum top cords")

A total of 86 sum top cords exist over 29 khipus with sum top cords

What is their range of sum top cords vs top cords vs double sum top cords?

Code

import sys
(khipu_dict, all_khipus) = kamayuq.fetch_khipus()

# Top Cord Range
min_top_cord = sys.maxsize
max_top_cord = -sys.maxsize - 1

for khipu_name in topcord_Fieldmark.matching_khipus():
    aKhipu = khipu_dict[khipu_name]
    topcords = aKhipu.top_cords()
    cord_vals = [cord.knotted_value() for cord in topcords]
    (min_khipu_top, max_khipu_top) = (min(cord_vals), max(cord_vals))
    (min_top_cord, max_top_cord) = (min(min_top_cord, min_khipu_top), max(max_top_cord, max_khipu_top))
print(f"Cord Value Range for Top Cords is [{min_top_cord} to {max_top_cord}]")

# Sum Top Cord Range
min_sum_top_cord = sys.maxsize
max_sum_top_cord = -sys.maxsize - 1

for khipu_name in sum_topcord_Fieldmark.matching_khipus():
    aKhipu = khipu_dict[khipu_name]
    topcord_sum_clusters = aKhipu.top_cord_sum_clusters()
    min_cord_vals = [aCluster.min_cord_value() for aCluster in topcord_sum_clusters]
    max_cord_vals = [aCluster.max_cord_value() for aCluster in topcord_sum_clusters]
    (min_khipu_top, max_khipu_top) = (min(min_cord_vals), max(max_cord_vals))
    (min_sum_top_cord, max_sum_top_cord) = (min(min_sum_top_cord, min_khipu_top), max(max_sum_top_cord, max_khipu_top))
print(f"Cord Value Range for Sum Top Cords is [{min_sum_top_cord} to {max_sum_top_cord}]")

# Double Sum Top Cord Range
# Sum Top Cord Range
min_double_sum_top_cord = sys.maxsize
max_double_sum_top_cord = -sys.maxsize - 1

for khipu_name in double_sum_topcord_Fieldmark.matching_khipus():
    aKhipu = khipu_dict[khipu_name]
    topcord_double_sum_clusters = aKhipu.top_cord_double_sum_clusters()
    min_cord_vals = [aCluster.min_cord_value() for aCluster in topcord_double_sum_clusters] or [min_double_sum_top_cord]
    max_cord_vals = [aCluster.max_cord_value() for aCluster in topcord_double_sum_clusters] or [max_double_sum_top_cord]
    (min_khipu_top, max_khipu_top) = (min(min_cord_vals), max(max_cord_vals))
    (min_double_sum_top_cord, max_double_sum_top_cord) = (min(min_double_sum_top_cord, min_khipu_top), max(max_double_sum_top_cord, max_khipu_top))
print(f"Cord Value Range for Double Sum Top Cords is [{min_double_sum_top_cord} to {max_double_sum_top_cord}]")

range_records = []
for khipu_name in sum_topcord_Fieldmark.matching_khipus():
    aKhipu = khipu_dict[khipu_name]
    khipu_min_cord_value = aKhipu.min_pendant_cord_value()
    khipu_max_cord_value = aKhipu.max_pendant_cord_value()
    khipu_mean_cord_value = aKhipu.mean_pendant_cord_value()
    range_records.append({'khipu_name': khipu_name, 
                          'min_cord_value': khipu_min_cord_value, 
                          'max_cord_value': khipu_max_cord_value, 
                          'mean_cord_value': khipu_mean_cord_value})
range_df = pd.DataFrame(range_records, 
                        columns=['khipu_name', 'min_cord_value', 'max_cord_value', 'mean_cord_value'])
max_total_cord_value = range_df['max_cord_value'].max()
mean_total_cord_value = round(range_df['mean_cord_value'].mean())
min_mean_total_cord_value = range_df['mean_cord_value'].min()
max_mean_total_cord_value = range_df['mean_cord_value'].max()
print(f"Sum TopCord Khipus: Mean Cord Values (Min:Mean:Max) = {min_mean_total_cord_value}:{mean_total_cord_value}:{max_mean_total_cord_value}")
print(f"Sum TopCord Khipus: Max Total Cord Value = {max_total_cord_value}")

Cord Value Range for Top Cords is [0 to 6678]
Cord Value Range for Sum Top Cords is [2 to 1417]
Cord Value Range for Double Sum Top Cords is [2 to 729]
Sum TopCord Khipus: Mean Cord Values (Min:Mean:Max) = 1:61:248
Sum TopCord Khipus: Max Total Cord Value = 2792

3. Khipus with Left/Right Ascher Sums - Either vs Both

Code

from fieldmark_ascher_pendant_pendant_sum import Fieldmark_Pendant_Pendant_Sum
pps_Fieldmark = Fieldmark_Pendant_Pendant_Sum()
pps_khipus = pps_Fieldmark.matching_khipus()
pps_df = pps_Fieldmark.dataframes[0].dataframe

from fieldmark_ascher_pendant_pendants_color_sum import Fieldmark_Pendant_Pendants_Color_Sum
pps_color_Fieldmark = Fieldmark_Pendant_Pendants_Color_Sum()
pps_color_khipus = pps_Fieldmark.matching_khipus()
pps_color_df = pps_color_Fieldmark.dataframes[0].dataframe

from fieldmark_ascher_indexed_pendant_sum import Fieldmark_Indexed_Pendant_Sum
indexed_pps_Fieldmark = Fieldmark_Indexed_Pendant_Sum()
indexed_pps_khipus = pps_Fieldmark.matching_khipus()
indexed_pps_df = indexed_pps_Fieldmark.dataframes[0].dataframe

from fieldmark_ascher_subsidiary_pendant_sum import Fieldmark_Subsidiary_Pendant_Sum
sub_pps_Fieldmark = Fieldmark_Subsidiary_Pendant_Sum()
sub_pps_khipus = pps_Fieldmark.matching_khipus()
sub_pps_df = sub_pps_Fieldmark.dataframes[0].dataframe

Code

def show_either_both_sums(df, df_name):
    has_either_mask = [(a > 0) or (b > 0) for (a,b) in zip(df['num_left_sums'].values.tolist(), df['num_right_sums'].values.tolist())]
    has_either_sum_df = df[has_either_mask]
    has_both_mask = [(a > 0) and (b > 0) for (a,b) in zip(df['num_left_sums'].values.tolist(), df['num_right_sums'].values.tolist())]
    has_both_sum_df = df[has_both_mask]
    print(f"{df_name} num either_sums = {len(has_either_sum_df)} num_both_sums = {len(has_both_sum_df)}")
    return (len(has_either_sum_df), len(has_both_sum_df))

either_pps, both_pps = show_either_both_sums(pps_df, "pps_df")
either_pps_color, both_pps_color = show_either_both_sums(pps_color_df, "pps_color_df")
either_indexed_pps, both_indexed_pps = show_either_both_sums(indexed_pps_df, "indexed_pps_df")
either_sub_pps, both_sub_pps = show_either_both_sums(sub_pps_df, "sub_pps_df")

sum_eithers = either_pps + either_pps_color + either_indexed_pps + either_sub_pps
sum_both = both_pps + both_pps_color + both_indexed_pps + both_sub_pps
print(f"sum_eithers = {sum_eithers} sum_both = {sum_both} ratio={round(100.0*float(sum_both)/float(sum_eithers))}%")

pps_df num either_sums = 427 num_both_sums = 325
pps_color_df num either_sums = 261 num_both_sums = 146
indexed_pps_df num either_sums = 228 num_both_sums = 118
sub_pps_df num either_sums = 117 num_both_sums = 59
sum_eithers = 1033 sum_both = 648 ratio=63%

4. White as the first cord of a cluster (any cluster)

Code

def num_clusters_with_white_first():
    num_clusters = 0
    num_white_leading_clusters = 0
    """ Calculate the total number of clusters in all khipus that start with a white cord """
    for aKhipu in all_khipus:
        primary_cord_clusters = aKhipu[:]
        num_clusters += len(primary_cord_clusters)
        first_cord_colors = [cluster.pendant_cords()[0].longest_ascher_color() for cluster in primary_cord_clusters if cluster.pendant_cords()]
        num_white_leading_clusters += sum([color == 'W'for color in first_cord_colors])
    return(num_white_leading_clusters, num_clusters)

(num_white_leading_clusters, num_clusters) = num_clusters_with_white_first()
print(f"Ratio of white leading clusters to all clusters = {round(100.0*float(num_white_leading_clusters)/float(num_clusters))}%")

Ratio of white leading clusters to all clusters = 32%

5. P Value for Pendant-pendant-sum handedness

How likely is the asymmetry of handedness (in this case 54%/46% right-handed/left-handed for all sums) compared to the null hypothesis of 50%/50%? One way to test is to use the Kolmogorov Smirnov Statistic, which follows a Kolmogorov distribution if the null hypothesis is true. If the observed data perfectly follow a normal distribution, the value of the KS statistic will be 0. The P-Value is used to decide whether the difference is large enough to reject the null hypothesis:

If the P-Value of the KS Test is larger than 0.05, we assume a normal distribution If the P-Value of the KS Test is smaller than 0.05, we do not assume a normal distribution

Clearly as we can see below, this is not a normal distribution with p being much less then < 0.01

Code

from fieldmark_ascher_pendant_pendant_sum import Fieldmark_Pendant_Pendant_Sum
pps_Fieldmark = Fieldmark_Pendant_Pendant_Sum()

all_khipus_df = pps_Fieldmark.dataframes[0].dataframe
pps_khipus_df = all_khipus_df[all_khipus_df.num_sum_cords > 0]
handedness_vector = pps_khipus_df.handedness.values.tolist()

from scipy.stats import kstest, norm
gaussian_null_data = norm.rvs(size=1000) # Generate some null-hypothesis data to test as well
(ks_statistic, p_value) = kstest(gaussian_null_data, 'norm')
print(f"Probability of null_hypothesis for null data = {round(100*p_value,3)}% ({p_value})")

(ks_statistic, p_value) = kstest(handedness_vector, 'norm')
print(f"Probability of null_hypothesis being true = {round(100.*p_value,3)}% ({p_value})")

Probability of null_hypothesis for null data = 70.105% (0.7010465408034103)
Probability of null_hypothesis being true = 0.0% (2.783198403904808e-75)

Code

from  scipy.stats import skew, kurtosis
print(f"Mean handedness = {round(np.mean(handedness_vector), 2)}")
print(f"Standard deviation handedness = {round(np.std(handedness_vector))}")
print(f"Skewness handedness = {round(skew(handedness_vector), 2)}")
print(f"Kurtosis handedness = {round(kurtosis(handedness_vector), 2)}")

Mean handedness = 2.28
Standard deviation handedness = 11
Skewness handedness = 1.01
Kurtosis handedness = 8.23

Code

from khipu import Khipu
(khipu_dict, all_khipus) = kamayuq.fetch_khipus()
num_knots = sum([aKhipu.num_knots() for aKhipu in all_khipus])
num_cords = sum([aKhipu.num_all_cords() for aKhipu in all_khipus]) # Returns ALL cords, including children of pendant cords, grandchildren, etc """
print(f"Number of knots = {num_knots} number of cords = {num_cords}")

Number of knots = 116246 number of cords = 55720