1/*
2MMJ's Cel Shader - v1.03
3----------------------------------------------------------------
4-- 180403 --
5This is a port of my old shader from around 2006 for Pete's OGL2
6plugin for ePSXe. It started out as a shader based on the
7"CComic" shader by Maruke. I liked his concept, but I was
8looking for something a little different in the output.
9
10Since the last release, I've seen some test screenshots from MAME
11using a port of my original shader and have also seen another
12port to get it working with the PCSX2 emulator. Having recently
13seen some Kingdom Hearts II and Soul Calibur 3 YouTube videos with
14my ported shader inspired me to revisit it and get it working in
15RetroArch.
16
17As for this version (1.03), I've made a few small modifications
18(such as to remove the OGL2Param references, which were specific
19to Pete's plugin) and I added some RetroArch Parameter support,
20so some values can now be changed in real time.
21
22Keep in mind, that this was originally developed for PS1, using
23various 3D games as a test. In general, it will look better in
24games with less detailed textures, as "busy" textures will lead
25to more outlining / messy appearance. Increasing "Outline
26Brightness" can help mitigate this some by lessening the
27"strength" of the outlines.
28
29Also (in regards to PS1 - I haven't really tested other systems
30too much yet), 1x internal resolution will look terrible. 2x
31will also probably be fairly blurry/messy-looking. For best
32results, you should probably stick to 4x or higher internal
33resolution with this shader.
34
35Parameters:
36-----------
37White Level Cutoff = Anything above this luminance value will be
38    forced to pure white.
39
40Black Level Cutoff = Anything below this luminance value will be
41    forced to pure black.
42
43Shading Levels = Determines how many color "slices" there should
44    be (not counting black/white cutoffs, which are always
45    applied).
46
47Saturation Modifier = Increase or decrease color saturation.
48    Default value boosts saturation a little for a more
49    cartoonish look. Set to 0.00 for grayscale.
50
51Outline Brightness = Adjusts darkness of the outlines. At a
52    setting of 1, outlines should be disabled.
53
54Shader Strength = Adjusts the weight of the color banding
55    portion of the shader from 0% (0.00) to 100% (1.00). At a
56    setting of 0.00, you can turn off the color banding effect
57    altogether, but still keep outlines enabled.
58-----------
59MMJuno
60*/
61
62// Parameter lines go here:
63#pragma parameter WhtCutoff "White Level Cutoff" 0.97 0.50 1.00 0.01
64#pragma parameter BlkCutoff "Black Level Cutoff" 0.03 0.00 0.50 0.01
65#pragma parameter ShdLevels "Shading Levels" 16.0 1.0 16.0 1.0
66#pragma parameter SatModify "Saturation Modifier" 1.32 0.00 2.00 0.01
67#pragma parameter OtlModify "Outline Brightness" 0.20 0.00 1.00 0.01
68#pragma parameter ShdWeight "Shader Strength" 0.50 0.00 1.00 0.01
69
70
71#if defined(VERTEX)
72
73#if __VERSION__ >= 130
74#define COMPAT_VARYING out
75#define COMPAT_ATTRIBUTE in
76#define COMPAT_TEXTURE texture
77#else
78#define COMPAT_VARYING varying
79#define COMPAT_ATTRIBUTE attribute
80#define COMPAT_TEXTURE texture2D
81#endif
82
83#ifdef GL_ES
84#define COMPAT_PRECISION mediump
85#else
86#define COMPAT_PRECISION
87#endif
88
89COMPAT_ATTRIBUTE vec4 VertexCoord;
90COMPAT_ATTRIBUTE vec4 COLOR;
91COMPAT_ATTRIBUTE vec4 TexCoord;
92COMPAT_VARYING vec4 COL0;
93COMPAT_VARYING vec4 TEX0;
94COMPAT_VARYING vec4 TEX1;
95COMPAT_VARYING vec4 TEX2;
96COMPAT_VARYING vec4 TEX3;
97
98vec4 _oPosition1;
99uniform mat4 MVPMatrix;
100uniform COMPAT_PRECISION int FrameDirection;
101uniform COMPAT_PRECISION int FrameCount;
102uniform COMPAT_PRECISION vec2 OutputSize;
103uniform COMPAT_PRECISION vec2 TextureSize;
104uniform COMPAT_PRECISION vec2 InputSize;
105
106// compatibility #defines
107#define vTexCoord TEX0.xy
108#define SourceSize vec4(TextureSize, 1.0 / TextureSize) //either TextureSize or InputSize
109#define OutSize vec4(OutputSize, 1.0 / OutputSize)
110
111void main()
112{
113    vec4 offset;
114
115    gl_Position = MVPMatrix * VertexCoord;
116
117    TEX0 = TexCoord.xyxy;
118
119    offset.xy = -(offset.zw = vec2(SourceSize.z, 0.0));
120    TEX1 = TEX0 + offset;
121
122    offset.xy = -(offset.zw = vec2(0.0, SourceSize.w));
123    TEX2 = TEX0 + offset;
124    TEX3 = TEX1 + offset;
125}
126
127#elif defined(FRAGMENT)
128
129#if __VERSION__ >= 130
130#define COMPAT_VARYING in
131#define COMPAT_TEXTURE texture
132out vec4 FragColor;
133#else
134#define COMPAT_VARYING varying
135#define FragColor gl_FragColor
136#define COMPAT_TEXTURE texture2D
137#endif
138
139#ifdef GL_ES
140#ifdef GL_FRAGMENT_PRECISION_HIGH
141precision highp float;
142#else
143precision mediump float;
144#endif
145#define COMPAT_PRECISION mediump
146#else
147#define COMPAT_PRECISION
148#endif
149
150uniform COMPAT_PRECISION int FrameDirection;
151uniform COMPAT_PRECISION int FrameCount;
152uniform COMPAT_PRECISION vec2 OutputSize;
153uniform COMPAT_PRECISION vec2 TextureSize;
154uniform COMPAT_PRECISION vec2 InputSize;
155uniform sampler2D Texture;
156COMPAT_VARYING vec4 TEX0;
157COMPAT_VARYING vec4 TEX1;
158COMPAT_VARYING vec4 TEX2;
159COMPAT_VARYING vec4 TEX3;
160
161// compatibility #defines
162#define Source Texture
163#define vTexCoord (TEX0.xy * TextureSize.xy / InputSize.xy)
164
165#define SourceSize vec4(TextureSize, 1.0 / TextureSize) //either TextureSize or InputSize
166#define OutSize vec4(OutputSize, 1.0 / OutputSize)
167
168#ifdef PARAMETER_UNIFORM
169uniform COMPAT_PRECISION float WhtCutoff;
170uniform COMPAT_PRECISION float BlkCutoff;
171uniform COMPAT_PRECISION float ShdLevels;
172uniform COMPAT_PRECISION float SatModify;
173uniform COMPAT_PRECISION float OtlModify;
174uniform COMPAT_PRECISION float ShdWeight;
175#else
176#define WhtCutoff 0.97
177#define BlkCutoff 0.03
178#define ShdLevels 9.0
179#define SatModify 1.15
180#define OtlModify 0.20
181#define ShdWeight 0.50
182#endif
183
184vec3 RGB2HSL(vec3 cRGB)
185{
186    float cR = cRGB[0], cG = cRGB[1], cB = cRGB[2];
187    float vMin = min(min(cR, cG), cB), vMax = max(max(cR, cG), cB);
188    float dMax = vMax - vMin, vS = 0.0, vH = 0.0, vL = (vMax + vMin) / 2.0;
189
190    // gray, no chroma
191    if(dMax == 0.0) {
192      vH = 0.0; vS = vH;
193
194    // chromatic data
195    } else {
196        if(vL < 0.5) { vS = dMax / (vMax + vMin); }
197        else         { vS = dMax / (2.0 - vMax - vMin); }
198
199        float dR = (((vMax - cR) * 0.1667) + (dMax * 0.5)) / dMax;
200        float dG = (((vMax - cG) * 0.1667) + (dMax * 0.5)) / dMax;
201        float dB = (((vMax - cB) * 0.1667) + (dMax * 0.5)) / dMax;
202
203        if     (cR >= vMax) { vH = dB - dG; }
204        else if(cG >= vMax) { vH = 0.3333 + dR - dB; }
205        else if(cB >= vMax) { vH = 0.6667 + dG - dR; }
206
207        if     (vH < 0.0) { vH += 1.0; }
208        else if(vH > 1.0) { vH -= 1.0; }
209    }
210    return vec3(vH, vS, vL);
211}
212
213float Hue2RGB(float v1, float v2, float vH)
214{
215    float v3 = 0.0;
216
217    if     (vH < 0.0) { vH += 1.0; }
218    else if(vH > 1.0) { vH -= 1.0; }
219
220    if     ((6.0 * vH) < 1.0) { v3 = v1 + (v2 - v1) * 6.0 * vH; }
221    else if((2.0 * vH) < 1.0) { v3 = v2; }
222    else if((3.0 * vH) < 2.0) { v3 = v1 + (v2 - v1) * (0.6667 - vH) * 6.0; }
223    else                      { v3 = v1; }
224
225    return v3;
226}
227
228vec3 HSL2RGB(vec3 vHSL)
229{
230    float cR = 0.0, cG = cR, cB = cR;
231
232    if(vHSL[1] == 0.0) {
233      cR = vHSL[2], cG = cR, cB = cR;
234
235    } else {
236        float v1 = 0.0, v2 = v1;
237
238        if(vHSL[2] < 0.5) { v2 = vHSL[2] * (1.0 + vHSL[1] ); }
239        else              { v2 = (vHSL[2] + vHSL[1] ) - (vHSL[1] * vHSL[2] ); }
240
241        v1 = 2.0 * vHSL[2] - v2;
242
243        cR = Hue2RGB(v1, v2, vHSL[0] + 0.3333);
244        cG = Hue2RGB(v1, v2, vHSL[0] );
245        cB = Hue2RGB(v1, v2, vHSL[0] - 0.3333);
246    }
247    return vec3(cR, cG, cB);
248}
249
250vec3 colorAdjust(vec3 cRGB)
251{
252    vec3 cHSL = RGB2HSL(cRGB);
253
254    float cr = 1.0 / ShdLevels;
255
256    // brightness modifier
257    float BrtModify = mod(cHSL[2], cr);
258
259    if     (cHSL[2] > WhtCutoff) { cHSL[1]  = 1.0; cHSL[2] = 1.0; }
260    else if(cHSL[2] > BlkCutoff) { cHSL[1] *= SatModify; cHSL[2] += (cHSL[2] * cr - BrtModify); }
261    else                         { cHSL[1]  = 0.0; cHSL[2] = 0.0; }
262    cRGB = 1.2 * HSL2RGB(cHSL);
263
264    return cRGB;
265}
266
267
268void main()
269{
270    vec3 c0 = COMPAT_TEXTURE(Source, TEX3.xy).rgb;
271    vec3 c1 = COMPAT_TEXTURE(Source, TEX2.xy).rgb;
272    vec3 c2 = COMPAT_TEXTURE(Source, TEX3.zy).rgb;
273    vec3 c3 = COMPAT_TEXTURE(Source, TEX1.xy).rgb;
274    vec3 c4 = COMPAT_TEXTURE(Source, TEX0.xy).rgb;
275    vec3 c5 = COMPAT_TEXTURE(Source, TEX1.zw).rgb;
276    vec3 c6 = COMPAT_TEXTURE(Source, TEX3.xw).rgb;
277    vec3 c7 = COMPAT_TEXTURE(Source, TEX2.zw).rgb;
278    vec3 c8 = COMPAT_TEXTURE(Source, TEX3.zw).rgb;
279
280    vec3 c9 = ((c0 + c2 + c6 + c8) * 0.15 + (c1 + c3 + c5 + c7) * 0.25 + c4) / 2.6;
281
282    vec3 o = vec3(1.0); vec3 h = vec3(0.05); vec3 hz = h; float k = 0.005;
283    float kz = 0.007; float i = 0.0;
284
285    vec3 cz = (c9 + h) / (dot(o, c9) + k);
286
287    hz = (cz - ((c0 + h) / (dot(o, c0) + k))); i  = kz / (dot(hz, hz) + kz);
288    hz = (cz - ((c1 + h) / (dot(o, c1) + k))); i += kz / (dot(hz, hz) + kz);
289    hz = (cz - ((c2 + h) / (dot(o, c2) + k))); i += kz / (dot(hz, hz) + kz);
290    hz = (cz - ((c3 + h) / (dot(o, c3) + k))); i += kz / (dot(hz, hz) + kz);
291    hz = (cz - ((c5 + h) / (dot(o, c5) + k))); i += kz / (dot(hz, hz) + kz);
292    hz = (cz - ((c6 + h) / (dot(o, c6) + k))); i += kz / (dot(hz, hz) + kz);
293    hz = (cz - ((c7 + h) / (dot(o, c7) + k))); i += kz / (dot(hz, hz) + kz);
294    hz = (cz - ((c8 + h) / (dot(o, c8) + k))); i += kz / (dot(hz, hz) + kz);
295
296    i /= 8.0; i = pow(i, 0.75);
297
298    if(i < OtlModify) { i = OtlModify; }
299    c9 = min(o, min(c9, c9 + dot(o, c9)));
300    FragColor.rgb = mix(c4 * i, colorAdjust(c9 * i), ShdWeight);
301}
302#endif
303