← All the evidencesentinel-personal-significance-experiment.py
SHA-256 9697dbc54ee5743b55d842cd7f8d0de6b341bcfa7ac9ae2cd9e7fc4ae8d5caef · download the original
#!/usr/bin/env python3
"""Synthetic demonstration of uncertain, changing personal significance.
No personal data, no actual Sentinel/ Foundation logs. Truth is used only by
simulator and offline evaluator. The inference methods observe numerical events.
Uses only Python standard library and a fixed seed.
"""
import math
import random
from collections import deque
SEED = 20261028
N = 24000
TRAIN = 8000
VALIDATION = 6000
TEST = N - TRAIN - VALIDATION
RNG = random.Random(SEED)
def clip(x, a=1e-8, b=1 - 1e-8):
return min(b, max(a, x))
def simulate(n=N):
preference = RNG.randrange(2)
records = []
switches = []
for t in range(n):
# Hidden current intention changes slowly and unpredictably.
if t and RNG.random() < 0.014:
preference = 1 - preference
switches.append(t)
constraint = int(RNG.random() < 0.36)
action = None if RNG.random() < 0.16 else (
preference if RNG.random() < (0.78 if not constraint else 0.14) else 1 - preference
)
# Explicit statements and corrections are observed numerical signals.
# Their semantic interpretation is SUPPLIED by this toy-world generator.
statement = None
correction = None
if RNG.random() < 0.037:
statement = preference if RNG.random() < 0.94 else 1 - preference
if RNG.random() < 0.025:
correction = preference if RNG.random() < 0.985 else 1 - preference
records.append((preference, constraint, action, statement, correction))
return records, switches
def predict(records, switch_hazard=0.018):
predictions = {'last_action': [], 'all_actions_window': [],
'last_statement': [], 'context_window': [],
'context_bayes': [], 'context_bayes_no_corrections': []}
all_window = deque(maxlen=21)
context_window = deque(maxlen=26)
last_action = None
last_explicit = None
belief = 0.5
belief_no_correction = 0.5
for _, constrained, action, statement, correction in records:
if action is not None:
last_action = action
all_window.append(action)
# Latest action ignores whether it was required.
predictions['last_action'].append(0.5 if last_action is None else
(0.75 if last_action else 0.25))
predictions['all_actions_window'].append(
(1 + sum(all_window)) / (2 + len(all_window)))
if statement is not None:
last_explicit = statement
if correction is not None:
last_explicit = correction
predictions['last_statement'].append(
0.5 if last_explicit is None else (0.92 if last_explicit else 0.08))
# A fair simple alternative: recent voluntary actions and explicit signals.
context_window.append((action if not constrained else None, statement, correction))
pos = neg = 1.0
for a, s, c in context_window:
for val, weight in ((a, 1), (s, 3), (c, 6)):
if val is not None:
if val:
pos += weight
else:
neg += weight
predictions['context_window'].append(pos / (pos + neg))
# Candidate: exact two-state filter under approximate, predeclared
# observation reliabilities. No latent labels are read here.
for is_full in (False, True):
prior = belief if is_full else belief_no_correction
prior = prior * (1 - switch_hazard) + (1 - prior) * switch_hazard
log_odds = math.log(clip(prior) / (1 - clip(prior)))
if action is not None and not constrained:
log_odds += (2 * action - 1) * math.log(.77 / .23)
if statement is not None:
log_odds += (2 * statement - 1) * math.log(.92 / .08)
if correction is not None and is_full:
log_odds += (2 * correction - 1) * math.log(.985 / .015)
posterior = 1 / (1 + math.exp(-max(-40, min(40, log_odds))))
if is_full:
belief = posterior
predictions['context_bayes'].append(posterior)
else:
belief_no_correction = posterior
predictions['context_bayes_no_corrections'].append(posterior)
return predictions
def score(labels, ps):
n = len(labels)
logloss = sum(-math.log2(clip(p if y else 1-p)) for y, p in zip(labels, ps)) / n
accuracy = sum((p >= 0.5) == bool(y) for y, p in zip(labels, ps)) / n
confidence = [(y, p) for y, p in zip(labels, ps) if p >= 0.90 or p <= 0.10]
wrong_conf = sum((p >= 0.5) != bool(y) for y, p in confidence)
return logloss, accuracy, len(confidence)/n, (wrong_conf/len(confidence) if confidence else float('nan'))
def adaptation(labels, ps, switch_points, start, end):
# Mean number of subsequent observations until a candidate has >=.7
# probability of the new goal for at least 3 consecutive observations.
times = []
for i in switch_points:
if i < start or i >= end - 45:
continue
target = labels[i]
found = 46
for k in range(45):
window = ps[i+k:i+k+3]
if all((p >= 0.70 if target else p <= 0.30) for p in window):
found = k
break
times.append(found)
return len(times), sum(times)/len(times) if times else float('nan')
def main():
rows, switches = simulate()
pred = predict(rows)
y = [r[0] for r in rows]
phases = [('validation', TRAIN, TRAIN+VALIDATION),
('test', TRAIN+VALIDATION, N)]
print('Sentinel personal significance candidate: synthetic, not real personal experience')
print('Seed:', SEED, 'Observations:', N,
'burn-in (no parameter fitting):', TRAIN, 'validation:', VALIDATION, 'untouched test:', TEST)
print('Hidden preference changes:',len(switches),
'constraint rate:',sum(r[1] for r in rows)/N,
'action missing rate:',sum(r[2] is None for r in rows)/N)
print('The hidden preference is available ONLY to the synthetic evaluator.')
print('Evidence model assumes visible constraint status, free-action reliability .77,')
print('statement reliability .92, correction reliability .985 and hazard .018.')
print('These are human-designed simulation assumptions, NOT learned values.')
for title,start,end in phases:
print('\n'+title.upper()+' (bits of log loss lower is better)')
print('%-30s %10s %10s %12s %13s %12s' % (
'model','logloss','accuracy','coverage90','error|conf90','switch-delay'))
for key, p in pred.items():
loss, acc, cover, err = score(y[start:end],p[start:end])
ns, delay = adaptation(y,p,switches,start,end)
print('%-30s %10.4f %10.4f %12.4f %13.4f %12.2f' %
(key,loss,acc,cover,err,delay))
print('Tested hidden preference switches:',len([i for i in switches if start <= i < end-45]))
# Source ablations and a constraint-free subset help check whether model
# improvement is due solely to seeing explicit corrections.
start = TRAIN + VALIDATION
mask = [i for i in range(start,N) if rows[i][3] is None and rows[i][4] is None]
print('\nTEST ROWS WITHOUT NEW EXPLICIT STATEMENT/CORRECTION:',len(mask))
for k in ('context_window','context_bayes','context_bayes_no_corrections'):
loss,acc,cover,err=score([y[i] for i in mask],[pred[k][i] for i in mask])
print('%-30s logloss=%.4f accuracy=%.4f coverage90=%.4f' % (k,loss,acc,cover))
print('\nAll scoring is on simulator labels, not independently assessed human priorities.')
print('This does not establish real-world preference discovery, causal importance, or consent.')
# Prespecified stress tests on the held-out sequence, with NO access to
# hidden preferences inside the learner. Time labels and emissions held fixed.
shuffle_rng = random.Random(SEED + 1)
constraints = [row[1] for row in rows]
shuffle_rng.shuffle(constraints)
distorted = [(y,c,a,s,k) for (y,_,a,s,k),c in zip(rows,constraints)]
stress = {
'shuffled constraint status': predict(distorted)['context_bayes'],
'very sticky old preference': predict(rows, switch_hazard=0.001)['context_bayes'],
'overreacts to changes': predict(rows, switch_hazard=0.09)['context_bayes'],
}
print('\nSENSITIVITY CHECKS on untouched test')
for name, p in stress.items():
loss, acc, cover, err=score(y[start:],p[start:])
print('%-30s logloss=%.4f accuracy=%.4f coverage90=%.4f' % (name,loss,acc,cover))
print('A correction rejecting an action must control permissions independently of these predictions.')
if __name__ == '__main__':
main()