Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486// SPDX-License-Identifier: GPL-2.0/* * Optimized string functions * * S390 version * Copyright IBM Corp. 2004 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) */
#define IN_ARCH_STRING_C 1#ifndef __NO_FORTIFY# define __NO_FORTIFY#endif
#include <linux/types.h>#include <linux/string.h>#include <linux/export.h>#include <asm/facility.h>#include <asm/asm.h>
#define SYMBOL_FUNCTION_ALIAS(alias, name) \asm(".globl " __stringify(alias) "\n\t" \ ".set " __stringify(alias) "," __stringify(name))
#ifdef __HAVE_ARCH_MEMMOVEnoinstr void *__memmove(void *dest, const void *src, size_t n){ const char *s = src; char *d = dest;
if (!n) return dest; if ((d <= s || d >= s + n)) { /* Forward copy */ while (n >= 256) { asm volatile( " mvc 0(256,%[d]),0(%[s])\n" : : [d] "a" (d), [s] "a" (s) : "memory"); d += 256; s += 256; n -= 256; } if (n) { asm volatile( " exrl %[n],0f\n" " j 1f\n" "0: mvc 0(1,%[d]),0(%[s])\n" "1:" : : [d] "a" (d), [s] "a" (s), [n] "a" (n - 1) : "memory"); } return dest; } /* Backward copy */ if (test_facility(61)) { /* Use mvcrl instruction if available */ while (n >= 256) { asm volatile( " lghi %%r0,255\n" " .insn sse,0xe50a00000000,%[d],%[s]\n" : [d] "=Q" (*(d + n - 256)) : [s] "Q" (*(s + n - 256)) : "0", "memory"); n -= 256; } if (n) { asm volatile( " lgr %%r0,%[n]\n" " .insn sse,0xe50a00000000,%[d],%[s]\n" : [d] "=Q" (*d) : [s] "Q" (*s), [n] "d" (n - 1) : "0", "memory"); } } else { while (n--) d[n] = s[n]; } return dest;}SYMBOL_FUNCTION_ALIAS(memmove, __memmove);EXPORT_SYMBOL(__memmove);EXPORT_SYMBOL(memmove);#endif
#ifdef __HAVE_ARCH_MEMSETnoinstr void *__memset(void *s, int c, size_t n){ char *xs = s;
if (!n) return s; if (!c) { /* Clear memory */ while (n >= 256) { asm volatile( " xc 0(256,%[xs]),0(%[xs])" : : [xs] "a" (xs) : "cc", "memory"); xs += 256; n -= 256; } if (!n) return s; asm volatile( " exrl %[n],0f\n" " j 1f\n" "0: xc 0(1,%[xs]),0(%[xs])\n" "1:" : : [xs] "a" (xs), [n] "a" (n - 1) : "cc", "memory"); } else { /* Fill memory */ while (n >= 256) { *xs = c; asm volatile( " mvc 1(255,%[xs]),0(%[xs])" : : [xs] "a" (xs) : "memory"); xs += 256; n -= 256; } if (!n) return s; *xs = c; if (n == 1) return s; asm volatile( " exrl %[n],0f\n" " j 1f\n" "0: mvc 1(1,%[xs]),0(%[xs])\n" "1:" : : [xs] "a" (xs), [n] "a" (n - 2) : "memory"); } return s;}SYMBOL_FUNCTION_ALIAS(memset, __memset);EXPORT_SYMBOL(__memset);EXPORT_SYMBOL(memset);#endif
#ifdef __HAVE_ARCH_MEMCPYnoinstr void *__memcpy(void *dest, const void *src, size_t n){ void *d = dest;
if (!n) return d; while (n >= 256) { asm volatile( " mvc 0(256,%[dest]),0(%[src])" : : [dest] "a" (dest), [src] "a" (src) : "memory"); dest += 256; src += 256; n -= 256; } if (!n) return d; asm volatile( " exrl %[n],1f\n" " j 2f\n" "1: mvc 0(1,%[dest]),0(%[src])\n" "2:" : : [dest] "a" (dest), [src] "a" (src), [n] "a" (n - 1) : "memory"); return d;}SYMBOL_FUNCTION_ALIAS(memcpy, __memcpy);EXPORT_SYMBOL(__memcpy);EXPORT_SYMBOL(memcpy);#endif
#define DEFINE_MEMSET(_bits, _bytes, _type) \void *__memset##_bits(_type *s, _type v, size_t n) \{ \ _type *xs = s; \ \ if (!n) \ return s; \ while (n >= 256) { \ *xs = v; \ asm volatile( \ " mvc %[_b](256-%[_b],%[xs]),0(%[xs])\n" \ : \ : [xs] "a" (xs), [_b] "i" (_bytes) \ : "memory"); \ xs = (_type *)((char *)xs + 256); \ n -= 256; \ } \ if (!n) \ return s; \ *xs = v; \ if (n == _bytes) \ return s; \ n -= _bytes + 1; \ asm volatile( \ " exrl %[n],1f\n" \ " j 2f\n" \ "1: mvc %[_b](1,%[xs]),0(%[xs])\n" \ "2:" \ : \ : [n] "a" (n), [xs] "a" (xs), [_b] "i" (_bytes) \ : "memory"); \ return s; \} \EXPORT_SYMBOL(__memset##_bits)
#ifdef __HAVE_ARCH_MEMSET16DEFINE_MEMSET(16, 2, uint16_t);#endif
#ifdef __HAVE_ARCH_MEMSET32DEFINE_MEMSET(32, 4, uint32_t);#endif
#ifdef __HAVE_ARCH_MEMSET64DEFINE_MEMSET(64, 8, uint64_t);#endif
/* * Helper functions to find the end of a string */static inline char *__strend(const char *s){ unsigned long e = 0;
asm volatile( " lghi 0,0\n" "0: srst %[e],%[s]\n" " jo 0b" : [e] "+&a" (e), [s] "+&a" (s) : : "cc", "memory", "0"); return (char *)e;}
static inline char *__strnend(const char *s, size_t n){ const char *p = s + n;
asm volatile( " lghi 0,0\n" "0: srst %[p],%[s]\n" " jo 0b" : [p] "+&d" (p), [s] "+&a" (s) : : "cc", "memory", "0"); return (char *)p;}
/** * strlen - Find the length of a string * @s: The string to be sized * * returns the length of @s */#ifdef __HAVE_ARCH_STRLENsize_t strlen(const char *s){ return __strend(s) - s;}EXPORT_SYMBOL(strlen);#endif
/** * strnlen - Find the length of a length-limited string * @s: The string to be sized * @n: The maximum number of bytes to search * * returns the minimum of the length of @s and @n */#ifdef __HAVE_ARCH_STRNLENsize_t strnlen(const char *s, size_t n){ return __strnend(s, n) - s;}EXPORT_SYMBOL(strnlen);#endif
/** * strcat - Append one %NUL-terminated string to another * @dest: The string to be appended to * @src: The string to append to it * * returns a pointer to @dest */#ifdef __HAVE_ARCH_STRCATchar *strcat(char *dest, const char *src){ unsigned long dummy = 0; char *ret = dest;
asm volatile( " lghi 0,0\n" "0: srst %[dummy],%[dest]\n" " jo 0b\n" "1: mvst %[dummy],%[src]\n" " jo 1b" : [dummy] "+&a" (dummy), [dest] "+&a" (dest), [src] "+&a" (src) : : "cc", "memory", "0"); return ret;}EXPORT_SYMBOL(strcat);#endif
/** * strncat - Append a length-limited, %NUL-terminated string to another * @dest: The string to be appended to * @src: The string to append to it * @n: The maximum numbers of bytes to copy * * returns a pointer to @dest */#ifdef __HAVE_ARCH_STRNCATchar *strncat(char *dest, const char *src, size_t n){ size_t len = __strnend(src, n) - src; char *p = __strend(dest);
p[len] = '\0'; memcpy(p, src, len); return dest;}EXPORT_SYMBOL(strncat);#endif
/** * strcmp - Compare two strings * @s1: One string * @s2: Another string * * returns 0 if @s1 and @s2 are equal, * < 0 if @s1 is less than @s2 * > 0 if @s1 is greater than @s2 */#ifdef __HAVE_ARCH_STRCMPint strcmp(const char *s1, const char *s2){ int ret = 0;
asm volatile( " lghi 0,0\n" "0: clst %[s1],%[s2]\n" " jo 0b\n" " je 1f\n" " ic %[ret],0(%[s1])\n" " ic 0,0(%[s2])\n" " sr %[ret],0\n" "1:" : [ret] "+&d" (ret), [s1] "+&a" (s1), [s2] "+&a" (s2) : : "cc", "memory", "0"); return ret;}EXPORT_SYMBOL(strcmp);#endif
static inline int clcle(const char *s1, unsigned long l1, const char *s2, unsigned long l2){ union register_pair r1 = { .even = (unsigned long)s1, .odd = l1, }; union register_pair r3 = { .even = (unsigned long)s2, .odd = l2, }; int cc;
asm volatile( "0: clcle %[r1],%[r3],0\n" " jo 0b\n" CC_IPM(cc) : CC_OUT(cc, cc), [r1] "+d" (r1.pair), [r3] "+d" (r3.pair) : : CC_CLOBBER_LIST("memory")); return CC_TRANSFORM(cc);}
/** * strstr - Find the first substring in a %NUL terminated string * @s1: The string to be searched * @s2: The string to search for */#ifdef __HAVE_ARCH_STRSTRchar *strstr(const char *s1, const char *s2){ int l1, l2;
l2 = __strend(s2) - s2; if (!l2) return (char *) s1; l1 = __strend(s1) - s1; while (l1-- >= l2) { int cc;
cc = clcle(s1, l2, s2, l2); if (!cc) return (char *) s1; s1++; } return NULL;}EXPORT_SYMBOL(strstr);#endif
/** * memchr - Find a character in an area of memory. * @s: The memory area * @c: The byte to search for * @n: The size of the area. * * returns the address of the first occurrence of @c, or %NULL * if @c is not found */#ifdef __HAVE_ARCH_MEMCHRvoid *memchr(const void *s, int c, size_t n){ const void *ret = s + n;
asm volatile( " lgr 0,%[c]\n" "0: srst %[ret],%[s]\n" " jo 0b\n" " jl 1f\n" " la %[ret],0\n" "1:" : [ret] "+&a" (ret), [s] "+&a" (s) : [c] "d" (c) : "cc", "memory", "0"); return (void *) ret;}EXPORT_SYMBOL(memchr);#endif
/** * memcmp - Compare two areas of memory * @s1: One area of memory * @s2: Another area of memory * @n: The size of the area. */#ifdef __HAVE_ARCH_MEMCMPint memcmp(const void *s1, const void *s2, size_t n){ int ret;
ret = clcle(s1, n, s2, n); if (ret) ret = ret == 1 ? -1 : 1; return ret;}EXPORT_SYMBOL(memcmp);#endif
/** * memscan - Find a character in an area of memory. * @s: The memory area * @c: The byte to search for * @n: The size of the area. * * returns the address of the first occurrence of @c, or 1 byte past * the area if @c is not found */#ifdef __HAVE_ARCH_MEMSCANvoid *memscan(void *s, int c, size_t n){ const void *ret = s + n;
asm volatile( " lgr 0,%[c]\n" "0: srst %[ret],%[s]\n" " jo 0b" : [ret] "+&a" (ret), [s] "+&a" (s) : [c] "d" (c) : "cc", "memory", "0"); return (void *)ret;}EXPORT_SYMBOL(memscan);#endif