Hugging Face's logo

Spaces:
docling-project
/
DeskForge
Running on Zero

App Files Community
DeskForge
File size: 8,920 Bytes
bf47338
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ec54602
156048d
 
 
 
bf47338
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
"""The "Try a screenshot" view: ground one instruction on your own screenshot.

Plain HTML + script inside one gr.HTML. The image is sent with the component's
`upload()` helper, and grounding runs through a server function, so no stock
Gradio widgets are involved. Example screenshots are served as static files.
"""

TEMPLATE = '<div class="dsh"></div>'

SCRIPT = r"""
const EX = props.examples || [], MODELS = props.models || [];
const st = { path: null, preview: null, text: '', model: MODELS[0], busy: false, out: null, err: '' };
const root = element.querySelector('.dsh') || element;
const esc = s => String(s ?? '').replace(/[&<>"]/g, c => ({'&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;'}[c]));

function render() {
  root.innerHTML = `
  <div class="dsh-grid">
    <div class="dsh-card">
      <div class="dsh-cap">Your screenshot</div>
      <label class="dsh-drop ${st.preview ? 'has' : ''}">
        ${st.preview ? `<img src="${esc(st.preview)}" alt="Screenshot">` :
          `<span class="dsh-drop-main">Drop, paste or click to choose a screenshot</span>
           <span class="dsh-drop-sub">PNG or JPG of a desktop, ideally a Linux desktop</span>`}
        <input type="file" accept="image/*" class="dsh-file" hidden>
      </label>
      <div class="dsh-cap">Instruction</div>
      <input class="dsh-text" type="text" placeholder="e.g. Open the File menu" value="${esc(st.text)}">
      <div class="dsh-row">
        <span class="dsh-seg">${MODELS.map(m => `<button type="button" data-model="${esc(m)}" class="${st.model === m ? 'on' : ''}">${esc(m)}</button>`).join('')}</span>
        <button type="button" class="dsh-go" data-act="go" ${st.busy ? 'disabled' : ''}>${st.busy ? 'Grounding…' : 'Ground it ↑'}</button>
      </div>
      ${st.err ? `<div class="dsh-err">${esc(st.err)}</div>` : ''}
    </div>
    <div class="dsh-card">
      <div class="dsh-cap">Where the model would click</div>
      ${st.out ? `<img class="dsh-result" src="${st.out.image}" alt="Predicted click">
        <div class="dsh-code">${esc(st.out.code)}</div>
        <div class="dsh-meta"><span>${esc(st.out.coords)}</span><span>${esc(st.out.time)}</span></div>`
      : `<div class="dsh-empty">${st.busy ? '<span class="dsh-spin"></span> Looking at the screen…' : 'The predicted click point shows up here.'}</div>`}
    </div>
  </div>
  <div class="dsh-cap dsh-excap">Or try an example</div>
  <div class="dsh-ex">${EX.map((e, i) => `<button type="button" class="dsh-exi" data-ex="${i}" title="${esc(e.text)}">
     <img src="${esc(e.url)}" alt=""><span>${esc(e.text)}</span></button>`).join('')}</div>`;
}

async function useFile(file) {
  if (!file || !file.type.startsWith('image/')) return;
  st.preview = URL.createObjectURL(file); st.out = null; st.err = ''; st.path = null; render();
  try { const r = await upload(file); st.path = r.path; }
  catch (e) { st.err = 'Upload failed, please try again.'; render(); }
}

async function go() {
  const text = (element.querySelector('.dsh-text') || {}).value || st.text;
  st.text = text;
  if (!st.path) { st.err = st.preview ? 'Still uploading, one moment…' : 'Choose a screenshot first.'; render(); return; }
  if (!text.trim()) { st.err = 'Write an instruction first.'; render(); return; }
  st.busy = true; st.err = ''; st.out = null; render();
  try { st.out = await gpuCall('ground_screenshot', { src: st.path, text, model: st.model }); }
  catch (e) {
    const msg = e && e.message ? e.message : String(e), f = friendlyGpuError(msg);
    st.err = f ? f.title + '. ' + f.text : 'Grounding failed: ' + msg;
  }
  st.busy = false; render();
}

element.addEventListener('change', e => { if (e.target.classList.contains('dsh-file')) useFile(e.target.files[0]); });
element.addEventListener('input', e => { if (e.target.classList.contains('dsh-text')) st.text = e.target.value; });
element.addEventListener('keydown', e => { if (e.target.classList.contains('dsh-text') && e.key === 'Enter') go(); });
element.addEventListener('dragover', e => e.preventDefault());
element.addEventListener('drop', e => { e.preventDefault(); useFile(e.dataTransfer.files[0]); });
element.addEventListener('paste', e => { const f = [...(e.clipboardData || {}).files || []][0]; if (f) useFile(f); });
element.addEventListener('click', e => {
  const t = e.target.closest('[data-model],[data-act],[data-ex]');
  if (!t) return;
  if (t.dataset.model) { st.model = t.dataset.model; render(); }
  else if (t.dataset.act === 'go') go();
  else if (t.dataset.ex) {
    const ex = EX[+t.dataset.ex];
    st.path = 'example:' + ex.name; st.preview = ex.url; st.text = ex.text; st.out = null; st.err = '';
    render(); go();
  }
});
render();
"""

CSS = """
#df-view-shot { display: none; background: #F3EDE2; padding: 22px 26px !important; }
#df-view-shot .dsh { font-family: 'Hanken Grotesk', system-ui, sans-serif; color: #2B2722; }
#df-view-shot .dsh-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
@media (max-width: 1000px) { .dsh-grid { grid-template-columns: 1fr; } }
#df-view-shot .dsh-card { background: #fff; border: 1px solid #E6E3DF; border-radius: 12px; padding: 14px; display: flex; flex-direction: column; gap: 9px;
            box-shadow: 0 1px 2px rgba(40,30,20,.04); }
#df-view-shot .dsh-cap { font: 500 10px 'JetBrains Mono', monospace; letter-spacing: .06em; text-transform: uppercase; color: #9a9389; }
#df-view-shot .dsh-drop { min-height: 230px; border: 1.5px dashed #DCD8D2; border-radius: 10px; display: flex; flex-direction: column; align-items: center;
            justify-content: center; gap: 4px; cursor: pointer; background: #FAF9F7; text-align: center; padding: 14px; overflow: hidden; }
#df-view-shot .dsh-drop:hover { border-color: #D2603A; }
#df-view-shot .dsh-drop.has { padding: 0; border-style: solid; background: #fff; }
#df-view-shot .dsh-drop img { width: 100%; display: block; }
#df-view-shot .dsh-drop-main { font-size: 14px; font-weight: 600; color: #3a332b; }
#df-view-shot .dsh-drop-sub { font-size: 12.5px; color: #8a8378; }
#df-view-shot .dsh-text { font: 14px 'Hanken Grotesk', system-ui, sans-serif; color: #2B2722; background: #fff; border: 1px solid #E6E3DF; border-radius: 9px;
            padding: 9px 11px; outline: none; }
#df-view-shot .dsh-text:focus { border-color: #D2603A; box-shadow: 0 0 0 3px rgba(210,96,58,.10); }
#df-view-shot .dsh-row { display: flex; align-items: center; justify-content: space-between; gap: 10px; flex-wrap: wrap; }
#df-view-shot .dsh-seg { display: inline-flex; border: 1px solid #E6E3DF; border-radius: 9px; overflow: hidden; background: #fff; }
#df-view-shot .dsh-seg button { font: 500 13px 'Hanken Grotesk', system-ui, sans-serif; color: #6f6a62; background: transparent; border: none; padding: 7px 12px; cursor: pointer; }
#df-view-shot .dsh-seg button + button { border-left: 1px solid #E6E3DF; }
#df-view-shot .dsh-seg button.on { background: #FBF3EC; color: #C0552F; font-weight: 600; }
#df-view-shot .dsh-go { font: 600 14px 'Hanken Grotesk', system-ui, sans-serif; color: #fff; background: #D2603A; border: 1px solid #C0552F; border-radius: 9px;
          padding: 8px 16px; cursor: pointer; }
#df-view-shot .dsh-go:disabled { opacity: .6; cursor: default; }
#df-view-shot .dsh-err { font-size: 13px; color: #A8452A; }
#df-view-shot .dsh-result { width: 100%; border-radius: 9px; border: 1px solid #E6E3DF; display: block; }
#df-view-shot .dsh-code { font: 500 12.5px/1.6 'JetBrains Mono', monospace; color: #3a332b; background: #FAF9F7; border: 1px solid #EEECE8; border-radius: 8px;
            padding: 8px 10px; overflow-wrap: anywhere; }
#df-view-shot .dsh-meta { display: flex; justify-content: space-between; font: 500 11.5px 'JetBrains Mono', monospace; color: #9a9389; }
#df-view-shot .dsh-empty { min-height: 230px; display: flex; align-items: center; justify-content: center; gap: 8px; border-radius: 10px; background: #FAF9F7;
             border: 1px solid #EEECE8; font-size: 13.5px; color: #8a8378; }
#df-view-shot .dsh-spin { width: 13px; height: 13px; border-radius: 50%; border: 2px solid #E6E3DF; border-top-color: #D2603A; animation: df-spin .7s linear infinite; }
#df-view-shot .dsh-excap { margin: 18px 0 8px; }
#df-view-shot .dsh-ex { display: grid; grid-template-columns: repeat(auto-fill, minmax(170px, 1fr)); gap: 10px; }
#df-view-shot .dsh-exi { display: flex; flex-direction: column; gap: 6px; background: #fff; border: 1px solid #E6E3DF; border-radius: 10px; padding: 6px; cursor: pointer;
           text-align: left; font: 12.5px/1.35 'Hanken Grotesk', system-ui, sans-serif; color: #3a332b; }
#df-view-shot .dsh-exi:hover { border-color: #D2603A; }
#df-view-shot .dsh-exi img { width: 100%; height: 92px; object-fit: cover; object-position: top left; border-radius: 6px; border: 1px solid #EEECE8; }
#df-view-shot .dsh-exi span { display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; }
"""