Synthetic face pool — 1000 StyleGAN headshots, ComfyUI hot-swap, 60x smaller thumbs
Worker cards now ship a real photo per person instead of monogram tiles:
- fetch_face_pool.py pulls 1000 faces from thispersondoesnotexist.com
- tag_face_pool.py runs deepface for gender/race/age, excludes <22yo
- manifest.jsonl: 952 servable, gender/race buckets populated
- /headshots/_thumbs/ pre-resized to 384px webp (587KB -> 11KB,
60x smaller; without this Chrome's parallel-connection budget
drops ~75% of tiles in a 40-card grid)
- /headshots/:key gender x race x age intersection bucketing with
gender-only fallback when intersection is sparse
- /headshots/generate/:key ComfyUI on-demand for the contractor
profile spotlight (cold ~1.5s, cached ~1ms; worker-derived
djb2 seed makes faces deterministic-per-worker but unique
across workers sharing the same prompt)
- serve_imagegen.py _cache_key() now includes seed (was caching
by prompt only -> 3 different worker seeds collapsed to 1
cached image; verified fix produces 3 distinct md5s)
- confidence-default name resolution: Xavier->man+hispanic,
Aisha->woman+black, etc. Every worker resolves to a bucket.
End-to-end: playwright run on /?q=forklift+operators+IL -> 21/21
cards loaded, 0 broken, all 384px webp.
Cache + binary pool gitignored; manifest tracked.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
tag_face_pool.py — run deepface gender + race classification over the
|
||||
synthetic face pool produced by fetch_face_pool.py and rewrite
|
||||
manifest.jsonl with `gender` (man / woman) and `race` (asian / black /
|
||||
hispanic / indian / middle_eastern / white) tags.
|
||||
|
||||
Run with the venv that has deepface installed:
|
||||
|
||||
/home/profit/.local/share/deepface-venv/bin/python \
|
||||
scripts/staffing/tag_face_pool.py
|
||||
|
||||
Idempotent: rows that already have BOTH gender and race tagged are
|
||||
skipped. Pass --force to re-tag everything.
|
||||
|
||||
Mapping deepface buckets → /headshots/ ?e= values:
|
||||
asian → split by manual region (deepface doesn't differentiate
|
||||
East / South Asian; we lump as 'east_asian' since the
|
||||
StyleGAN training set leans East Asian)
|
||||
indian → south_asian
|
||||
middle eastern → middle_eastern
|
||||
black → black
|
||||
hispanic → hispanic
|
||||
white → caucasian
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
DEEPFACE_RACE_TO_HINT = {
|
||||
"asian": "east_asian",
|
||||
"indian": "south_asian",
|
||||
"middle eastern": "middle_eastern",
|
||||
"black": "black",
|
||||
"latino hispanic": "hispanic",
|
||||
"hispanic": "hispanic",
|
||||
"white": "caucasian",
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument(
|
||||
"--out",
|
||||
default=os.path.join(os.path.dirname(__file__), "..", "..", "data", "headshots"),
|
||||
)
|
||||
p.add_argument("--force", action="store_true", help="re-tag rows that already have gender+race")
|
||||
p.add_argument("--limit", type=int, default=0, help="cap how many faces to process this run (0 = all)")
|
||||
p.add_argument("--min-age", type=int, default=22, help="exclude faces estimated below this age (kids/teens). Staffing context = legal-age workers only.")
|
||||
args = p.parse_args()
|
||||
|
||||
out = os.path.realpath(args.out)
|
||||
manifest_path = os.path.join(out, "manifest.jsonl")
|
||||
if not os.path.exists(manifest_path):
|
||||
print(f"manifest not found: {manifest_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"loading deepface (cold start ~10-15s for first model build)…")
|
||||
from deepface import DeepFace # type: ignore
|
||||
|
||||
rows = []
|
||||
with open(manifest_path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
rows.append(json.loads(line))
|
||||
print(f"manifest: {len(rows)} rows")
|
||||
|
||||
todo = [
|
||||
r for r in rows
|
||||
if args.force or r.get("gender") is None or r.get("race") is None or r.get("age") is None
|
||||
]
|
||||
if args.limit > 0:
|
||||
todo = todo[: args.limit]
|
||||
print(f"to tag: {len(todo)} faces")
|
||||
|
||||
if not todo:
|
||||
print("nothing to do.")
|
||||
return
|
||||
|
||||
counts_g = {}
|
||||
counts_r = {}
|
||||
failed = 0
|
||||
t0 = time.time()
|
||||
for i, r in enumerate(todo):
|
||||
full = os.path.join(out, r["file"])
|
||||
try:
|
||||
ana = DeepFace.analyze(
|
||||
img_path=full,
|
||||
actions=["gender", "race", "age"],
|
||||
enforce_detection=False,
|
||||
silent=True,
|
||||
)
|
||||
if isinstance(ana, list):
|
||||
ana = ana[0] if ana else {}
|
||||
g_raw = (ana.get("dominant_gender") or "").lower().strip()
|
||||
r["gender"] = (
|
||||
"man" if g_raw.startswith("man") else
|
||||
"woman" if g_raw.startswith("woman") else
|
||||
None
|
||||
)
|
||||
r_raw = (ana.get("dominant_race") or "").lower().strip()
|
||||
r["race"] = DEEPFACE_RACE_TO_HINT.get(r_raw, None)
|
||||
if r["race"] is None and r_raw:
|
||||
r["race_raw"] = r_raw
|
||||
# Age estimation — exclude minors / teens. Staffing context
|
||||
# uses adult workers only. Threshold is 22 by default
|
||||
# (legal + a buffer because age estimation is noisy).
|
||||
try:
|
||||
age = int(round(float(ana.get("age") or 0)))
|
||||
except Exception:
|
||||
age = 0
|
||||
r["age"] = age
|
||||
if age and age < args.min_age:
|
||||
r["excluded"] = "minor"
|
||||
else:
|
||||
r.pop("excluded", None)
|
||||
counts_g[r["gender"] or "unknown"] = counts_g.get(r["gender"] or "unknown", 0) + 1
|
||||
counts_r[r["race"] or r_raw or "unknown"] = counts_r.get(r["race"] or r_raw or "unknown", 0) + 1
|
||||
except Exception as e:
|
||||
r["tag_error"] = f"{type(e).__name__}: {e}"
|
||||
failed += 1
|
||||
if (i + 1) % 25 == 0 or (i + 1) == len(todo):
|
||||
elapsed = time.time() - t0
|
||||
rate = (i + 1) / elapsed if elapsed > 0 else 0
|
||||
eta = (len(todo) - i - 1) / rate if rate > 0 else 0
|
||||
print(f" [{i+1}/{len(todo)}] rate={rate:.1f}/s eta={eta:.0f}s failed={failed}")
|
||||
print(f" gender: {counts_g}")
|
||||
print(f" race : {counts_r}")
|
||||
|
||||
# Write updated manifest atomically
|
||||
tmp = manifest_path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
for r in rows:
|
||||
f.write(json.dumps(r) + "\n")
|
||||
os.replace(tmp, manifest_path)
|
||||
|
||||
final_g = {}
|
||||
final_r = {}
|
||||
excluded = 0
|
||||
age_hist = {"<18": 0, "18-22": 0, "22-30": 0, "30-40": 0, "40-50": 0, "50-60": 0, "60+": 0, "unknown": 0}
|
||||
for r in rows:
|
||||
if r.get("excluded"):
|
||||
excluded += 1
|
||||
continue
|
||||
final_g[r.get("gender") or "untagged"] = final_g.get(r.get("gender") or "untagged", 0) + 1
|
||||
final_r[r.get("race") or "untagged"] = final_r.get(r.get("race") or "untagged", 0) + 1
|
||||
a = r.get("age") or 0
|
||||
if a == 0: age_hist["unknown"] += 1
|
||||
elif a < 18: age_hist["<18"] += 1
|
||||
elif a < 22: age_hist["18-22"] += 1
|
||||
elif a < 30: age_hist["22-30"] += 1
|
||||
elif a < 40: age_hist["30-40"] += 1
|
||||
elif a < 50: age_hist["40-50"] += 1
|
||||
elif a < 60: age_hist["50-60"] += 1
|
||||
else: age_hist["60+"] += 1
|
||||
print(f"\nDone. {len(rows)} rows, {excluded} excluded as <{args.min_age}, {failed} tag errors, {time.time()-t0:.1f}s")
|
||||
print(f" final gender: {final_g}")
|
||||
print(f" final race : {final_r}")
|
||||
print(f" age dist : {age_hist}")
|
||||
print(f"\nNext: poke /headshots/__reload to refresh the in-memory pool.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user