Upload folder using huggingface_hub

2216aae verified 10 months ago

3.21 kB

from __future__ import annotations

import itertools
import re
import textwrap
from typing import TYPE_CHECKING


if TYPE_CHECKING:
    from collections.abc import Mapping, Sequence


# match $identifier or ${identifier} and replace with value in env
# If this identifier is at the beginning of whitespace on a line
# and its value is a list then it is treated as
# block substitution by indenting to that depth and putting each element
# of the list on its own line
# if the identifier is on a line starting with non-whitespace and a list
# then it is comma separated ${,foo} will insert a comma before the list
# if this list is not empty and ${foo,} will insert one after.


class CodeTemplate:
    substitution_str = r"(^[^\n\S]*)?\$([^\d\W]\w*|\{,?[^\d\W]\w*\,?})"
    substitution = re.compile(substitution_str, re.MULTILINE)

    pattern: str
    filename: str

    @staticmethod
    def from_file(filename: str) -> CodeTemplate:
        with open(filename) as f:
            return CodeTemplate(f.read(), filename)

    def __init__(self, pattern: str, filename: str = "") -> None:
        self.pattern = pattern
        self.filename = filename

    def substitute(
        self, env: Mapping[str, object] | None = None, **kwargs: object
    ) -> str:
        if env is None:
            env = {}

        def lookup(v: str) -> object:
            assert env is not None
            return kwargs[v] if v in kwargs else env[v]

        def indent_lines(indent: str, v: Sequence[object]) -> str:
            content = "\n".join(
                itertools.chain.from_iterable(str(e).splitlines() for e in v)
            )
            content = textwrap.indent(content, prefix=indent)
            # Remove trailing whitespace on each line
            return "\n".join(map(str.rstrip, content.splitlines())).rstrip()

        def replace(match: re.Match[str]) -> str:
            indent = match.group(1)
            key = match.group(2)
            comma_before = ""
            comma_after = ""
            if key[0] == "{":
                key = key[1:-1]
                if key[0] == ",":
                    comma_before = ", "
                    key = key[1:]
                if key[-1] == ",":
                    comma_after = ", "
                    key = key[:-1]
            v = lookup(key)
            if indent is not None:
                if not isinstance(v, list):
                    v = [v]
                return indent_lines(indent, v)
            elif isinstance(v, list):
                middle = ", ".join([str(x) for x in v])
                if len(v) == 0:
                    return middle
                return comma_before + middle + comma_after
            else:
                return str(v)

        return self.substitution.sub(replace, self.pattern)


if __name__ == "__main__":
    c = CodeTemplate(
        """\
    int foo($args) {
        $bar
            $bar
        $a+$b
    }
    int commatest(int a${,stuff})
    int notest(int a${,empty,})
    """
    )
    print(
        c.substitute(
            args=["hi", 8],
            bar=["what", 7],
            a=3,
            b=4,
            stuff=["things...", "others"],
            empty=[],
        )
    )

	from __future__ import annotations

	import itertools
	import re
	import textwrap
	from typing import TYPE_CHECKING


	if TYPE_CHECKING:
	from collections.abc import Mapping, Sequence


	# match $identifier or ${identifier} and replace with value in env
	# If this identifier is at the beginning of whitespace on a line
	# and its value is a list then it is treated as
	# block substitution by indenting to that depth and putting each element
	# of the list on its own line
	# if the identifier is on a line starting with non-whitespace and a list
	# then it is comma separated ${,foo} will insert a comma before the list
	# if this list is not empty and ${foo,} will insert one after.


	class CodeTemplate:
	substitution_str = r"(^[^\n\S])?\$([^\d\W]\w\|\{,?[^\d\W]\w*\,?})"
	substitution = re.compile(substitution_str, re.MULTILINE)

	pattern: str
	filename: str

	@staticmethod
	def from_file(filename: str) -> CodeTemplate:
	with open(filename) as f:
	return CodeTemplate(f.read(), filename)

	def __init__(self, pattern: str, filename: str = "") -> None:
	self.pattern = pattern
	self.filename = filename

	def substitute(
	self, env: Mapping[str, object] \| None = None, **kwargs: object
	) -> str:
	if env is None:
	env = {}

	def lookup(v: str) -> object:
	assert env is not None
	return kwargs[v] if v in kwargs else env[v]

	def indent_lines(indent: str, v: Sequence[object]) -> str:
	content = "\n".join(
	itertools.chain.from_iterable(str(e).splitlines() for e in v)
	)
	content = textwrap.indent(content, prefix=indent)
	# Remove trailing whitespace on each line
	return "\n".join(map(str.rstrip, content.splitlines())).rstrip()

	def replace(match: re.Match[str]) -> str:
	indent = match.group(1)
	key = match.group(2)
	comma_before = ""
	comma_after = ""
	if key[0] == "{":
	key = key[1:-1]
	if key[0] == ",":
	comma_before = ", "
	key = key[1:]
	if key[-1] == ",":
	comma_after = ", "
	key = key[:-1]
	v = lookup(key)
	if indent is not None:
	if not isinstance(v, list):
	v = [v]
	return indent_lines(indent, v)
	elif isinstance(v, list):
	middle = ", ".join([str(x) for x in v])
	if len(v) == 0:
	return middle
	return comma_before + middle + comma_after
	else:
	return str(v)

	return self.substitution.sub(replace, self.pattern)


	if __name__ == "__main__":
	c = CodeTemplate(
	"""\
	int foo($args) {

	$bar
	$bar
	$a+$b
	}
	int commatest(int a${,stuff})
	int notest(int a${,empty,})
	"""
	)
	print(
	c.substitute(
	args=["hi", 8],
	bar=["what", 7],
	a=3,
	b=4,
	stuff=["things...", "others"],
	empty=[],
	)
	)